@kirigami/php-prepros 1.0.9 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +300 -59
- package/index.js +1 -1
- package/package.json +4 -3
- package/src/libraries/arr.class.php +29 -0
- package/src/libraries/cache.class.php +5 -4
- package/src/libraries/curl.class.php +106 -0
- package/src/libraries/html.class.php +2 -0
- package/src/libraries/img.class.php +1 -1
- package/src/libraries/prepros.class.php +22 -3
- package/src/libraries/scraper.class.php +320 -0
- package/src/libraries/std.class.php +6 -2
- package/src/libraries/str.class.php +34 -0
- package/src/phpjs/mount.js +28 -0
- package/src/prepros.js +93 -24
- package/src/prepros.php +4 -17
- package/src/runenv.php +17 -0
- package/src/utils/findfile.js +25 -0
- package/src/utils.inc.php +21 -2
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
<?php
|
|
2
|
+
|
|
3
|
+
class CURL {
|
|
4
|
+
|
|
5
|
+
const HEADERS = [
|
|
6
|
+
'User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36',
|
|
7
|
+
'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
|
|
8
|
+
'Accept-Language: fr-CA,fr;q=0.9,en-US;q=0.8,en;q=0.7',
|
|
9
|
+
'Sec-Ch-Ua: "Not_A Brand";v="99", "Google Chrome";v="109", "Chromium";v="109"',
|
|
10
|
+
'Sec-Ch-Ua-Mobile: ?0',
|
|
11
|
+
'Sec-Ch-Ua-Platform: "Windows"',
|
|
12
|
+
'Sec-Fetch-Dest: document',
|
|
13
|
+
'Sec-Fetch-Mode: navigate',
|
|
14
|
+
'Sec-Fetch-Site: none',
|
|
15
|
+
'Sec-Fetch-User: ?1',
|
|
16
|
+
'Upgrade-Insecure-Requests: 1',
|
|
17
|
+
];
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
private static function getCookiePath() {
|
|
21
|
+
return '/project/.cookie.txt';
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
public static function urlExists(string $url, $mimereg = null)
|
|
26
|
+
{
|
|
27
|
+
if(!$url) return false;
|
|
28
|
+
if(!STR::is_url($url)) return false;
|
|
29
|
+
if(!$info = self::getInfo($url)) return false;
|
|
30
|
+
// print_r($info);
|
|
31
|
+
// if($mimereg && !empty($info['content_type']) && !preg_match('#' . $mimereg . '#i', $info['content_type'])) return false;
|
|
32
|
+
return ($info['http_code'] >= 200 && $info['http_code'] < 400);
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
public static function getInfo(string $url) {
|
|
37
|
+
if(!STR::is_url($url)) return false;
|
|
38
|
+
$cookiePath = self::getCookiePath();
|
|
39
|
+
if(!is_file($cookiePath)) touch($cookiePath);
|
|
40
|
+
$ch = curl_init($url);
|
|
41
|
+
curl_setopt_array($ch,[
|
|
42
|
+
CURLOPT_NOBODY => true,
|
|
43
|
+
CURLOPT_FOLLOWLOCATION => true,
|
|
44
|
+
CURLOPT_TIMEOUT => 15,
|
|
45
|
+
CURLOPT_SSL_VERIFYPEER => false,
|
|
46
|
+
CURLOPT_SSL_VERIFYHOST => false,
|
|
47
|
+
CURLOPT_ENCODING => 'gzip, deflate',
|
|
48
|
+
CURLOPT_COOKIEFILE => $cookiePath,
|
|
49
|
+
CURLOPT_COOKIEJAR => $cookiePath,
|
|
50
|
+
CURLOPT_HTTPHEADER => self::HEADERS,
|
|
51
|
+
]);
|
|
52
|
+
curl_exec($ch);
|
|
53
|
+
$info = curl_getinfo($ch);
|
|
54
|
+
PREPROS::exportFile($cookiePath);
|
|
55
|
+
return $info;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
public static function getContents(string $file, $dest = null, $clb = null)
|
|
60
|
+
{
|
|
61
|
+
$cookiePath = self::getCookiePath();
|
|
62
|
+
if(!is_file($cookiePath)) touch($cookiePath);
|
|
63
|
+
$chnd = curl_init($file);
|
|
64
|
+
curl_setopt_array($chnd, [
|
|
65
|
+
CURLOPT_AUTOREFERER => true,
|
|
66
|
+
CURLOPT_FOLLOWLOCATION => true,
|
|
67
|
+
CURLOPT_SSL_VERIFYHOST => false,
|
|
68
|
+
CURLOPT_SSL_VERIFYPEER => false,
|
|
69
|
+
CURLOPT_TIMEOUT => 60,
|
|
70
|
+
CURLOPT_CONNECTTIMEOUT => 10,
|
|
71
|
+
CURLOPT_ENCODING => 'gzip,deflate',
|
|
72
|
+
CURLOPT_NOPROGRESS => ($clb ? false : true),
|
|
73
|
+
CURLOPT_RETURNTRANSFER => ($dest ? false : true),
|
|
74
|
+
CURLOPT_COOKIEFILE => $cookiePath,
|
|
75
|
+
CURLOPT_COOKIEJAR => $cookiePath,
|
|
76
|
+
CURLOPT_HTTPHEADER => self::HEADERS,
|
|
77
|
+
]);
|
|
78
|
+
if ($clb) {
|
|
79
|
+
$lastprog = -1;
|
|
80
|
+
curl_setopt($chnd, CURLOPT_PROGRESSFUNCTION, function ($chnd, $totalbytes, $downbytes, $expupbytes, $upbytes) use (&$lastprog, $clb) {
|
|
81
|
+
if ($totalbytes > 0) {
|
|
82
|
+
$prog = $downbytes / $totalbytes;
|
|
83
|
+
} else $prog = 0;
|
|
84
|
+
$prog = round($prog, 5);
|
|
85
|
+
if ($prog != $lastprog) {
|
|
86
|
+
$lastprog = $prog;
|
|
87
|
+
call_user_func($clb, $prog);
|
|
88
|
+
}
|
|
89
|
+
});
|
|
90
|
+
}
|
|
91
|
+
if ($dest) {
|
|
92
|
+
if (!$fhnd = fopen($dest, "wb")) return false;
|
|
93
|
+
curl_setopt($chnd, CURLOPT_WRITEFUNCTION, function ($chnd, $data) use (&$fhnd) {
|
|
94
|
+
return fwrite($fhnd, $data);
|
|
95
|
+
});
|
|
96
|
+
}
|
|
97
|
+
$results = curl_exec($chnd);
|
|
98
|
+
$info = curl_getinfo($chnd);
|
|
99
|
+
if (!empty($fhnd)) fclose($fhnd);
|
|
100
|
+
if (!in_array($info['http_code'], [200, 201])) $results = false;
|
|
101
|
+
PREPROS::exportFile($cookiePath);
|
|
102
|
+
return $dest ? ($results !== false) : $results;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
}
|
|
@@ -70,6 +70,7 @@ class HTML
|
|
|
70
70
|
}
|
|
71
71
|
|
|
72
72
|
if ($isRaw) {
|
|
73
|
+
/** @var DOMElement $node */
|
|
73
74
|
$inner = trim($node->innerHTML, "\n\r");
|
|
74
75
|
if (trim($inner) === '') {
|
|
75
76
|
return "{$pad}<{$tag}{$attrs}></{$tag}>\n";
|
|
@@ -212,6 +213,7 @@ class HTML
|
|
|
212
213
|
private static function renderAttrs(Dom\Node $node): string
|
|
213
214
|
{
|
|
214
215
|
$out = '';
|
|
216
|
+
/** @var DOMElement $node */
|
|
215
217
|
if (!$node->hasAttributes()) {
|
|
216
218
|
return $out;
|
|
217
219
|
}
|
|
@@ -116,7 +116,7 @@ class IMG
|
|
|
116
116
|
'webp' => (function_exists('imagewebp') ? imagewebp($this->im, $dest, 85) : false),
|
|
117
117
|
default => throw new Exception("Invalid output file type.")
|
|
118
118
|
};
|
|
119
|
-
PREPROS::exportFile(realpath($dest));
|
|
119
|
+
// PREPROS::exportFile(realpath($dest));
|
|
120
120
|
return $this;
|
|
121
121
|
}
|
|
122
122
|
}
|
|
@@ -105,10 +105,13 @@ final class PREPROS
|
|
|
105
105
|
}
|
|
106
106
|
|
|
107
107
|
|
|
108
|
-
public static function exportFile(string $file): void
|
|
108
|
+
public static function exportFile(string|array $file): void
|
|
109
109
|
{
|
|
110
|
-
if
|
|
111
|
-
|
|
110
|
+
if(is_string($file)) $file = [$file];
|
|
111
|
+
foreach($file as $value) {
|
|
112
|
+
if (!$path = realpath($value)) continue;
|
|
113
|
+
self::$files[] = $path;
|
|
114
|
+
}
|
|
112
115
|
}
|
|
113
116
|
|
|
114
117
|
|
|
@@ -120,6 +123,22 @@ final class PREPROS
|
|
|
120
123
|
}
|
|
121
124
|
|
|
122
125
|
|
|
126
|
+
private static function callJS(string $cmd, array $params=[]): object
|
|
127
|
+
{
|
|
128
|
+
$args = ['command' => $cmd] + $params;
|
|
129
|
+
/** @disregard [1010] Optional explanation */
|
|
130
|
+
$results = json_decode(post_message_to_js(json_encode($args)));
|
|
131
|
+
return $results;
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
public static function mount(string|array $patterns) {
|
|
136
|
+
$results = self::callJS('mount', ['patterns' => $patterns]);
|
|
137
|
+
if(!$results->success) return false;
|
|
138
|
+
return $results->results;
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
|
|
123
142
|
public static function registerHook(string $hook, callable $clb)
|
|
124
143
|
{
|
|
125
144
|
self::$hooks[$hook][] = $clb;
|
|
@@ -0,0 +1,320 @@
|
|
|
1
|
+
<?php
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class SCRAPER
|
|
5
|
+
{
|
|
6
|
+
|
|
7
|
+
private static function scrape(string $url) {
|
|
8
|
+
if(!STR::is_url($url)) throw new Exception("Url is not valid.");
|
|
9
|
+
if(!$contents = self::getContents($url)) throw new Exception("Url crawling seems to have been block.");
|
|
10
|
+
if(!$xpath = self::loadHTML($contents)) throw new Exception("Can't parse server response.");
|
|
11
|
+
|
|
12
|
+
$metas = new stdClass;
|
|
13
|
+
$metas->url = $url;
|
|
14
|
+
$metas->title = '';
|
|
15
|
+
$metas->description = '';
|
|
16
|
+
$metas->image = '';
|
|
17
|
+
$metas->label = '';
|
|
18
|
+
|
|
19
|
+
if(($results = $xpath->query('//script[@type="application/ld+json"]')) && $results->length) {
|
|
20
|
+
$graphs = [];
|
|
21
|
+
foreach($results as $item) {
|
|
22
|
+
if($ld = json_decode(trim($item->textContent))) {
|
|
23
|
+
if(empty($ld->{'@graph'})) $graphs[] = $ld;
|
|
24
|
+
else foreach($ld->{'@graph'} as $item) $graphs[] = $item;
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
foreach($graphs as $graph) {
|
|
28
|
+
if(empty($graph->{'@type'})) continue;
|
|
29
|
+
if(is_array($graph->{'@type'}) || strtolower($graph->{'@type'}) != 'imageobject') continue;
|
|
30
|
+
if(!empty($graph->url)) $metas->image = STR::html_entities_decode($graph->url);
|
|
31
|
+
}
|
|
32
|
+
foreach($graphs as $graph) {
|
|
33
|
+
if(empty($graph->{'@type'})) continue;
|
|
34
|
+
if(is_array($graph->{'@type'}) || strtolower($graph->{'@type'}) != 'website') continue;
|
|
35
|
+
if(!empty($graph->name)) $metas->label = STR::html_entities_decode($graph->name);
|
|
36
|
+
}
|
|
37
|
+
foreach($graphs as $graph) {
|
|
38
|
+
if(empty($graph->{'@type'})) continue;
|
|
39
|
+
if(is_array($graph->{'@type'}) || strtolower($graph->{'@type'}) != 'organization') continue;
|
|
40
|
+
if(!empty($graph->name)) $metas->label = STR::html_entities_decode($graph->name);
|
|
41
|
+
}
|
|
42
|
+
foreach($graphs as $graph) {
|
|
43
|
+
if(empty($graph->{'@type'})) continue;
|
|
44
|
+
if(is_array($graph->{'@type'}) || strtolower($graph->{'@type'}) != 'newsmediaorganization') continue;
|
|
45
|
+
if(!empty($graph->name)) $metas->label = STR::html_entities_decode($graph->name);
|
|
46
|
+
}
|
|
47
|
+
foreach($graphs as $graph) {
|
|
48
|
+
if(empty($graph->{'@type'})) continue;
|
|
49
|
+
if(is_array($graph->{'@type'}) || strtolower($graph->{'@type'}) != 'webpage') continue;
|
|
50
|
+
if(!empty($graph->name)) $metas->title = STR::html_entities_decode($graph->name);
|
|
51
|
+
if(!empty($graph->description)) $metas->description = STR::html_entities_decode($graph->description);
|
|
52
|
+
if(!empty($graph->image->url)) $metas->image = STR::html_entities_decode($graph->image->url);
|
|
53
|
+
if(!empty($graph->thumbnailUrl)) $metas->image = STR::html_entities_decode($graph->thumbnailUrl);
|
|
54
|
+
if(!empty($graph->publisher->name)) $metas->label = STR::html_entities_decode($graph->publisher->name);
|
|
55
|
+
if(!empty($graph->mainEntity->name)) $metas->title = STR::html_entities_decode($graph->mainEntity->name);
|
|
56
|
+
if(!empty($graph->mainEntity->image->url)) $metas->image = STR::html_entities_decode($graph->mainEntity->image->url);
|
|
57
|
+
}
|
|
58
|
+
foreach($graphs as $graph) {
|
|
59
|
+
if(empty($graph->{'@type'})) continue;
|
|
60
|
+
if(is_array($graph->{'@type'}) || strtolower($graph->{'@type'}) != 'article') continue;
|
|
61
|
+
if(!empty($graph->headline)) $metas->title = STR::html_entities_decode($graph->headline);
|
|
62
|
+
if(!empty($graph->description)) $metas->description = STR::html_entities_decode($graph->description);
|
|
63
|
+
if(!empty($graph->publisher->name)) $metas->label = STR::html_entities_decode($graph->publisher->name);
|
|
64
|
+
}
|
|
65
|
+
foreach($graphs as $graph) {
|
|
66
|
+
if(empty($graph->{'@type'})) continue;
|
|
67
|
+
if(is_array($graph->{'@type'}) || strtolower($graph->{'@type'}) != 'newsarticle') continue;
|
|
68
|
+
if(!empty($graph->headline)) $metas->title = STR::html_entities_decode($graph->headline);
|
|
69
|
+
if(!empty($graph->description)) $metas->description = STR::html_entities_decode($graph->description);
|
|
70
|
+
if(!empty($graph->publisher->name)) $metas->label = STR::html_entities_decode($graph->publisher->name);
|
|
71
|
+
if(!empty($graph->thumbnailUrl)) $metas->image = STR::html_entities_decode($graph->thumbnailUrl);
|
|
72
|
+
if(!empty($graph->image->url)) $metas->image = STR::html_entities_decode($graph->image->url);
|
|
73
|
+
}
|
|
74
|
+
foreach($graphs as $graph) {
|
|
75
|
+
if(empty($graph->{'@type'})) continue;
|
|
76
|
+
if(is_array($graph->{'@type'}) || strtolower($graph->{'@type'}) != 'opinionnewsarticle') continue;
|
|
77
|
+
if(!empty($graph->headline)) $metas->title = STR::html_entities_decode($graph->headline);
|
|
78
|
+
if(!empty($graph->description)) $metas->description = STR::html_entities_decode($graph->description);
|
|
79
|
+
if(!empty($graph->publisher->name)) $metas->label = STR::html_entities_decode($graph->publisher->name);
|
|
80
|
+
if(!empty($graph->thumbnailUrl)) $metas->image = STR::html_entities_decode($graph->thumbnailUrl);
|
|
81
|
+
if(!empty($graph->image->url)) $metas->image = STR::html_entities_decode($graph->image->url);
|
|
82
|
+
}
|
|
83
|
+
foreach($graphs as $graph) {
|
|
84
|
+
if(empty($graph->{'@type'})) continue;
|
|
85
|
+
if(is_array($graph->{'@type'}) || strtolower($graph->{'@type'}) != 'radioepisode') continue;
|
|
86
|
+
if(!empty($graph->name)) $metas->title = STR::html_entities_decode($graph->name);
|
|
87
|
+
if(!empty($graph->description)) $metas->description = STR::html_entities_decode($graph->description);
|
|
88
|
+
if(!empty($graph->productionCompany->name)) $metas->label = STR::html_entities_decode($graph->productionCompany->name);
|
|
89
|
+
if(!empty($graph->thumbnailUrl)) $metas->image = STR::html_entities_decode($graph->thumbnailUrl);
|
|
90
|
+
if(!empty($graph->image->url)) $metas->image = STR::html_entities_decode($graph->image->url);
|
|
91
|
+
}
|
|
92
|
+
foreach($graphs as $graph) {
|
|
93
|
+
if(empty($graph->{'@type'})) continue;
|
|
94
|
+
if(is_array($graph->{'@type'}) || strtolower($graph->{'@type'}) != 'videogame') continue;
|
|
95
|
+
if(!empty($graph->name)) $metas->title = STR::html_entities_decode($graph->name);
|
|
96
|
+
if(!empty($graph->description)) $metas->description = STR::html_entities_decode($graph->description);
|
|
97
|
+
if(!empty($graph->image)) $metas->image = STR::html_entities_decode($graph->image);
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
if(($results = $xpath->query('//meta[@name="dc.title"]')) && $results->length) $metas->title = getFirstItemAttr($results, 'content');
|
|
102
|
+
if(($results = $xpath->query('//meta[@name="dcterms.title"]')) && $results->length) $metas->title = getFirstItemAttr($results, 'content');
|
|
103
|
+
if(($results = $xpath->query('//meta[@name="twitter:title"]')) && $results->length) $metas->title = getFirstItemAttr($results, 'content');
|
|
104
|
+
if(($results = $xpath->query('//meta[@property="og:title"]')) && $results->length) $metas->title = getFirstItemAttr($results, 'content');
|
|
105
|
+
if(($results = $xpath->query('//shreddit-title')) && $results->length) $metas->title = getFirstItemAttr($results, 'title');
|
|
106
|
+
if(($results = $xpath->query('//meta[@name="dc.description"]')) && $results->length) $metas->description = getFirstItemAttr($results, 'content');
|
|
107
|
+
if(($results = $xpath->query('//meta[@name="dcterms.description"]')) && $results->length) $metas->description = getFirstItemAttr($results, 'content');
|
|
108
|
+
if(($results = $xpath->query('//meta[@name="twitter:description"]')) && $results->length) $metas->description = getFirstItemAttr($results, 'content');
|
|
109
|
+
if(($results = $xpath->query('//meta[@property="og:description"]')) && $results->length) $metas->description = getFirstItemAttr($results, 'content');
|
|
110
|
+
if(($results = $xpath->query('//meta[@name="description"]')) && $results->length) $metas->description = getFirstItemAttr($results, 'content');
|
|
111
|
+
if(($results = $xpath->query('//meta[@name="twitter:image"]')) && $results->length) $metas->image = getFirstItemAttr($results, 'content');
|
|
112
|
+
if(($results = $xpath->query('//meta[@property="og:image"]')) && $results->length) $metas->image = getFirstItemAttr($results, 'content');
|
|
113
|
+
if(($results = $xpath->query('//meta[@name="og:image"]')) && $results->length) $metas->image = getFirstItemAttr($results, 'content');
|
|
114
|
+
if(($results = $xpath->query('//img[@rel="og:image rdfs:seeAlso"]')) && $results->length) $metas->image = getFirstItemAttr($results, 'src');
|
|
115
|
+
if(($results = $xpath->query('//link[@rel="canonical"]')) && $results->length) $metas->url = getFirstItemAttr($results, 'href');
|
|
116
|
+
if(($results = $xpath->query('//meta[@name="twitter:url"]')) && $results->length) $metas->url = getFirstItemAttr($results, 'content');
|
|
117
|
+
if(($results = $xpath->query('//meta[@property="og:url"]')) && $results->length) $metas->url = getFirstItemAttr($results, 'content');
|
|
118
|
+
|
|
119
|
+
if(empty($metas->label) && ($results = $xpath->query('//meta[@name="twitter:app:name:iphone"]')) && $results->length) $metas->label = getFirstItemAttr($results, 'content');
|
|
120
|
+
if(empty($metas->label) && ($results = $xpath->query('//meta[@name="twitter:app:name:googleplay"]')) && $results->length) $metas->label = getFirstItemAttr($results, 'content');
|
|
121
|
+
if(empty($metas->label) && ($results = $xpath->query('//meta[@name="og:site_name"]')) && $results->length) $metas->label = getFirstItemAttr($results, 'content');
|
|
122
|
+
if(empty($metas->label) && ($results = $xpath->query('//meta[@property="og:site_name"]')) && $results->length) $metas->label = getFirstItemAttr($results, 'content');
|
|
123
|
+
if(empty($metas->label) && ($results = $xpath->query('//meta[@name="dc.publisher"]')) && $results->length) $metas->label = getFirstItemAttr($results, 'content');
|
|
124
|
+
if(empty($metas->label) && ($results = $xpath->query('//meta[@name="DC.Publisher"]')) && $results->length) $metas->label = getFirstItemAttr($results, 'content');
|
|
125
|
+
if(empty($metas->label) && ($results = $xpath->query('//meta[@name="publisher"]')) && $results->length) $metas->label = getFirstItemAttr($results, 'content');
|
|
126
|
+
if(empty($metas->label) && ($results = $xpath->query('//meta[@property="article:author"]')) && $results->length) $metas->label = getFirstItemAttr($results, 'content');
|
|
127
|
+
if(empty($metas->label) && ($results = $xpath->query('//meta[@name="author"]')) && $results->length) $metas->label = getFirstItemAttr($results, 'content');
|
|
128
|
+
if(empty($metas->label) && ($results = $xpath->query('//meta[@name="twitter:creator"]')) && $results->length) $metas->label = getFirstItemAttr($results, 'content');
|
|
129
|
+
if(empty($metas->label) && ($results = $xpath->query('//meta[@name="dcterms.creator"]')) && $results->length) $metas->label = getFirstItemAttr($results, 'content');
|
|
130
|
+
if(empty($metas->label) && ($results = $xpath->query('//meta[@property="al:ios:app_name"]')) && $results->length) $metas->label = getFirstItemAttr($results, 'content');
|
|
131
|
+
if(empty($metas->label) && ($results = $xpath->query('//meta[@property="al:android:app_name"]')) && $results->length) $metas->label = getFirstItemAttr($results, 'content');
|
|
132
|
+
if(empty($metas->label) && ($results = $xpath->query('//meta[@name="DC.publisher"]')) && $results->length) $metas->label = getFirstItemAttr($results, 'content');
|
|
133
|
+
if(empty($metas->label) && ($results = $xpath->query('//span[@class="footer__signature"]')) && $results->length) $metas->label = $results->item(0)->textContent;
|
|
134
|
+
if(empty($metas->label) && ($results = $xpath->query('//p[@class="site-title"]')) && $results->length) $metas->label = trim($results->item(0)->textContent);
|
|
135
|
+
if(empty($metas->label) && ($results = $xpath->query('//meta[@name="copyright"]')) && $results->length) $metas->label = trim(getFirstItemAttr($results, 'content'), '© ');
|
|
136
|
+
if(empty($metas->label) && preg_match("#\{'blogId': '[0-9]+', 'title': '([^']+)'#i", $contents, $m)) $metas->label = $m[1];
|
|
137
|
+
|
|
138
|
+
if(($results = $xpath->query('//link[@rel="alternate" and @type="application/json+oembed" and @href]')) && $results->length) {
|
|
139
|
+
if(STR::is_url(($oembedurl = getFirstItemAttr($results, 'href')))) {
|
|
140
|
+
if($json = self::getContents($oembedurl)) {
|
|
141
|
+
if($oem = json_decode($json)) {
|
|
142
|
+
if(empty($metas->title) && !empty($oem->title)) $metas->title = $oem->title;
|
|
143
|
+
if(empty($metas->description) && !empty($oem->summary)) $metas->description = $oem->summary;
|
|
144
|
+
if(empty($metas->label) && !empty($oem->provider_name)) $metas->label = $oem->provider_name;
|
|
145
|
+
if(empty($metas->image) && !empty($oem->thumbnail_url)) $metas->image = $oem->thumbnail_url;
|
|
146
|
+
if(empty($metas->image) && !empty($oem->url)) $metas->image = $oem->url;
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
if(($results = $xpath->query('//script[@id="__UNIVERSAL_DATA_FOR_REHYDRATION__" and @type="application/json"]')) && $results->length) {
|
|
152
|
+
if($data = json_decode($results->item(0)->textContent)) {
|
|
153
|
+
if(!empty($data->{'__DEFAULT_SCOPE__'}->{'webapp.video-detail'}->itemInfo->itemStruct)) {
|
|
154
|
+
$data = $data->{'__DEFAULT_SCOPE__'}->{'webapp.video-detail'}->itemInfo->itemStruct;
|
|
155
|
+
if(!empty($data->desc)) $metas->description = $data->desc;
|
|
156
|
+
if(!empty($data->author->nickname)) $metas->title = $data->author->nickname;
|
|
157
|
+
if(!empty($data->video->cover)) $metas->image = $data->video->cover;
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
if(($results = $xpath->query('//script[@type="application/json" and @data-sjs]')) && $results->length) {
|
|
162
|
+
foreach($results as $item) {
|
|
163
|
+
if(!$data = @json_decode($item->textContent)) continue;
|
|
164
|
+
if($owner = ARR::find_key($data, 'owner_as_page')) {
|
|
165
|
+
if(!empty($owner->name)) $metas->label = $owner->name;
|
|
166
|
+
}
|
|
167
|
+
if($route = ARR::find_key($data, 'route')) {
|
|
168
|
+
if(!empty($route->meta->title)) $metas->title = $route->meta->title;
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
if(empty($metas->label)) {
|
|
173
|
+
if(($results = $xpath->query('//link[@rel="manifest" and @href]')) && $results->length) {
|
|
174
|
+
if($manifesturl = get_absolute_url($url, getFirstItemAttr($results, 'href'))) {
|
|
175
|
+
if($json = self::getContents($manifesturl)) {
|
|
176
|
+
if($manifest = json_decode($json)) {
|
|
177
|
+
if(!empty($manifest->name)) $metas->label = $manifest->name;
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
if(empty($metas->label) && (($results = $xpath->query('//meta[@property="fb:page_id"]')) && $results->length)) {
|
|
184
|
+
if($_contents = self::getContents('https://facebook.com/' . getFirstItemAttr($results, 'content'))) {
|
|
185
|
+
if($xp = self::loadHTML($_contents)) {
|
|
186
|
+
if(($results = $xp->query('//meta[@property="og:title"]')) && $results->length) {
|
|
187
|
+
$metas->label = getFirstItemAttr($results, 'content');
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
if(empty($metas->label) && preg_match('#Drupal\.settings,(\{.*?\})\);#i', $contents, $m))
|
|
193
|
+
if($data = json_decode($m[1]))
|
|
194
|
+
if(!empty($data->jquery_ajax_load->site_name))
|
|
195
|
+
$metas->label = $data->jquery_ajax_load->site_name;
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
if(empty($metas->image) && ($results = $xpath->query('//span[@class="background-image not-mobile"]')) && $results->length)
|
|
199
|
+
if(preg_match('#url\(\'(.*?)\'\)#i', getFirstItemAttr($results, 'style'), $m))
|
|
200
|
+
$metas->image = $m[1];
|
|
201
|
+
|
|
202
|
+
if(empty($metas->image) && ($results = $xpath->query('//link[@rel="icon" and @type="image/jpeg"]')) && $results->length) $metas->image = getFirstItemAttr($results, 'href');
|
|
203
|
+
if(empty($metas->image) && ($results = $xpath->query('//link[@rel="icon" and @type="image/png"]')) && $results->length) $metas->image = getFirstItemAttr($results, 'href');
|
|
204
|
+
if(empty($metas->image) && ($results = $xpath->query('//article//img')) && $results->length) $metas->image = getFirstItemAttr($results, 'src');
|
|
205
|
+
if(empty($metas->image) && ($results = $xpath->query('//div[@id="image_box"]//img')) && $results->length) $metas->image = getFirstItemAttr($results, 'src');
|
|
206
|
+
if(empty($metas->image) && ($results = $xpath->query('//link[@rel="shortcut icon" and @type="image/jpeg" and @href]')) && $results->length) $metas->image = getFirstItemAttr($results, 'href');
|
|
207
|
+
if(empty($metas->image) && ($results = $xpath->query('//link[@rel="apple-touch-icon" and @sizes]')) && $results->length) {
|
|
208
|
+
$size = 0;
|
|
209
|
+
foreach($results as $item) {
|
|
210
|
+
/** @var DOMElement $item */
|
|
211
|
+
$nsize = current(explode('x', $item->getAttribute('sizes')));
|
|
212
|
+
if($nsize > $size) {
|
|
213
|
+
$metas->image = $item->getAttribute('href');
|
|
214
|
+
$size = $nsize;
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
if(empty($metas->image) && ($results = $xpath->query('//link[@rel="apple-touch-icon"]')) && $results->length) $metas->image = getFirstItemAttr($results, 'href');
|
|
219
|
+
if(empty($metas->title) && ($results = $xpath->query('//title')) && $results->length) $metas->title = $results->item(0)->textContent;
|
|
220
|
+
if(empty($metas->label)) { $parts = explode('|', $metas->title); if(count($parts) > 1) $metas->label = trim(array_pop($parts)); }
|
|
221
|
+
if(empty($metas->label)) { $parts = explode(' - ', $metas->title); if(count($parts) > 1) $metas->label = trim(array_pop($parts)); }
|
|
222
|
+
if(strpos($metas->label, '@') === 0) $metas->label = ucwords(preg_replace('#^@#i', '', $metas->label));
|
|
223
|
+
|
|
224
|
+
if(empty($metas->label) && ($results = $xpath->query('//div[@id="elevate-global" and @data-elevate-global]')) && $results->length)
|
|
225
|
+
if($data = json_decode(getFirstItemAttr($results, 'data-elevate-global')))
|
|
226
|
+
if(!empty($data->content_organization))
|
|
227
|
+
$metas->label = $data->content_organization;
|
|
228
|
+
|
|
229
|
+
if(empty($metas->label) && preg_match('#_spPageContextInfo=(\{.*?\});#i', $contents, $m))
|
|
230
|
+
if($data = json_decode($m[1]))
|
|
231
|
+
if(!empty($data->webTitle))
|
|
232
|
+
$metas->label = $data->webTitle;
|
|
233
|
+
|
|
234
|
+
if(empty($metas->description) && ($results = $xpath->query('//p')) && $results->length){
|
|
235
|
+
foreach($results as $item) {
|
|
236
|
+
if(!($desc = trim($item->textContent))) continue;
|
|
237
|
+
if(count(explode(' ', $desc)) < 5) continue;
|
|
238
|
+
$metas->description = $desc;
|
|
239
|
+
break;
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
if(strpos($metas->image, '//') === 0) $metas->image = 'https:' . $metas->image;
|
|
244
|
+
if(!empty($metas->image) && !parse_url($metas->image, PHP_URL_SCHEME)) $metas->image = get_absolute_url($url, $metas->image);
|
|
245
|
+
if(parse_url($metas->image, PHP_URL_HOST) == 'lookaside.fbsbx.com')
|
|
246
|
+
if($_contents = self::getContents($metas->image))
|
|
247
|
+
if(preg_match('#location\.href = "(.*?)";#i', $_contents, $m))
|
|
248
|
+
if($_contents = self::getContents(str_replace('\\/', '/', $m[1])))
|
|
249
|
+
if($xp = self::loadHTML($_contents))
|
|
250
|
+
if(($results = $xp->query('//link[@rel="preload" and @as="image" and contains(@href, ".jpg")]')) && $results->length)
|
|
251
|
+
$metas->image = getFirstItemAttr($results, 'href');
|
|
252
|
+
|
|
253
|
+
if(!url_exists($metas->image, '^image/') && ($results = $xpath->query('//img[@src]')) && $results->length) {
|
|
254
|
+
foreach($results as $item) {
|
|
255
|
+
/** @var DOMElement $item */
|
|
256
|
+
if(in_array(pathinfo(parse_url($item->getAttribute('src'), PHP_URL_PATH), PATHINFO_EXTENSION), ['jpg', 'jpeg', 'png', 'webp'])) {
|
|
257
|
+
$metas->image = $item->getAttribute('src');
|
|
258
|
+
if(strpos($metas->image, '//') === 0) $metas->image = 'https:' . $metas->image;
|
|
259
|
+
if(!empty($metas->image) && !parse_url($metas->image, PHP_URL_SCHEME)) $metas->image = get_absolute_url($url, $metas->image);
|
|
260
|
+
break;
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
$metas->url = urldecode($metas->url);
|
|
266
|
+
if($metas->label) {
|
|
267
|
+
$parts = preg_split('/\s*\|\s*/', $metas->title);
|
|
268
|
+
foreach($parts as $k => $v) if(STR::slug($v) == STR::slug($metas->label)) unset($parts[$k]);
|
|
269
|
+
$metas->title = join(' | ', $parts);
|
|
270
|
+
$parts = preg_split('/\s+\-\s+/', $metas->title);
|
|
271
|
+
foreach($parts as $k => $v) if(STR::slug($v) == STR::slug($metas->label)) unset($parts[$k]);
|
|
272
|
+
$metas->title = join(' - ', $parts);
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
$metas->label = preg_replace('#^www\.#i', '', trim(current(explode(', ', current(explode(' — ', current(explode(' - ', current(explode(' | ', $metas->label))))))))));
|
|
276
|
+
$metas->description = trim(strip_tags($metas->description));
|
|
277
|
+
return $metas;
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
private static function getContents($url) {
|
|
282
|
+
if(($contents = CACHE::get(($key = 'scraper_' . STR::shorthash($url)))) !== null) return $contents;
|
|
283
|
+
$contents = CURL::getContents($url);
|
|
284
|
+
CACHE::set($key, $contents);
|
|
285
|
+
return $contents;
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
private static function loadHTML(string $contents) {
|
|
290
|
+
$dom = Dom\HTMLDocument::createFromString($contents, LIBXML_NOERROR | Dom\HTML_NO_DEFAULT_NS);
|
|
291
|
+
return new DOM\Xpath($dom);
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
public static function get(string $url) {
|
|
296
|
+
if(($metas = CACHE::get(($key = 'metas_' . STR::shorthash($url)))) !== null) return $metas;
|
|
297
|
+
if(!$metas = self::scrape($url)) throw new Exception("Can't crawl Url.");
|
|
298
|
+
if(!trim($metas->title)) return false;
|
|
299
|
+
$metas->key = $key;
|
|
300
|
+
CACHE::set($key, $metas);
|
|
301
|
+
return $metas;
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
|
|
307
|
+
function getFirstItemAttr(\Dom\Node|\Dom\NodeList $results, string $attr) {
|
|
308
|
+
/** @var DOM\Element $item */
|
|
309
|
+
$item = $results->item(0);
|
|
310
|
+
return $item->getAttribute($attr);
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
|
|
314
|
+
function url_exists(string $url, string|null $mimereg = null) {
|
|
315
|
+
if(($exists = CACHE::get(($key = 'urlexists_' . STR::shorthash($url)))) !== null) return $exists->response;
|
|
316
|
+
$exists = new stdClass;
|
|
317
|
+
$exists->response = CURL::urlExists($url, $mimereg);
|
|
318
|
+
CACHE::set($key, $exists);
|
|
319
|
+
return $exists->response;
|
|
320
|
+
}
|
|
@@ -2,14 +2,17 @@
|
|
|
2
2
|
|
|
3
3
|
class STD {
|
|
4
4
|
|
|
5
|
+
const RESULT_PATH = '/internal/prepros_result.json';
|
|
5
6
|
|
|
6
7
|
public static function succeed($props = []): void
|
|
7
8
|
{
|
|
8
9
|
$return = new stdClass;
|
|
9
10
|
$return->success = true;
|
|
11
|
+
$return->files = PREPROS::getExportedFiles();
|
|
10
12
|
if(is_string($props)) $return->message = $props;
|
|
11
13
|
else foreach($props as $k => $v) $return->{$k} = $v;
|
|
12
|
-
|
|
14
|
+
if(!is_dir(pathinfo(self::RESULT_PATH, PATHINFO_DIRNAME))) mkdir(pathinfo(self::RESULT_PATH, PATHINFO_DIRNAME), true, 0777);
|
|
15
|
+
file_put_contents(self::RESULT_PATH, json_encode($return, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE), FILE_APPEND);
|
|
13
16
|
exit(0);
|
|
14
17
|
}
|
|
15
18
|
|
|
@@ -18,9 +21,10 @@ class STD {
|
|
|
18
21
|
{
|
|
19
22
|
$return = new stdClass;
|
|
20
23
|
$return->success = false;
|
|
24
|
+
$return->files = PREPROS::getExportedFiles();
|
|
21
25
|
if(is_string($props)) $return->error = $props;
|
|
22
26
|
else foreach($props as $k => $v) $return->{$k} = $v;
|
|
23
|
-
file_put_contents(
|
|
27
|
+
file_put_contents(self::RESULT_PATH, json_encode($return, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE), FILE_APPEND);
|
|
24
28
|
exit(1);
|
|
25
29
|
}
|
|
26
30
|
|
|
@@ -44,4 +44,38 @@ class STR
|
|
|
44
44
|
}
|
|
45
45
|
return implode("\n", $lines);
|
|
46
46
|
}
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
public static function is_url(string $str): bool
|
|
50
|
+
{
|
|
51
|
+
if (!$sheme = strtolower(@parse_url($str, PHP_URL_SCHEME))) return false;
|
|
52
|
+
return in_array($sheme, ['http', 'https', 'itunes', 'ftp', 'ftps', 'ssh', 'ssl', 'sftp']);
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
public static function html_entities_decode(string $str): string
|
|
57
|
+
{
|
|
58
|
+
// $str = html_entity_decode(trim($str), ENT_QUOTES, 'UTF-8');
|
|
59
|
+
// $str = htmlentities($str, ENT_QUOTES, 'UTF-8');
|
|
60
|
+
// $str = mb_convert_encoding($str, 'HTML-ENTITIES', 'UTF-8');
|
|
61
|
+
$str = html_entity_decode(trim($str), ENT_QUOTES, 'UTF-8');
|
|
62
|
+
// $str = htmlentities($str, ENT_QUOTES, 'UTF-8');
|
|
63
|
+
return $str;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
public static function shorthash(string$str): string
|
|
68
|
+
{
|
|
69
|
+
return substr(hash('sha256', $str), 0, 12);
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
public static function slug(string $str): string
|
|
74
|
+
{
|
|
75
|
+
$str = iconv('UTF-8', 'ASCII//TRANSLIT//IGNORE', $str);
|
|
76
|
+
$str = strtolower($str);
|
|
77
|
+
$str = preg_replace('/[^a-z0-9]/', '', $str);
|
|
78
|
+
return $str;
|
|
79
|
+
}
|
|
80
|
+
|
|
47
81
|
}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import fs from 'fs';
|
|
2
|
+
import path from "path";
|
|
3
|
+
import findFiles from "../utils/findfile.js";
|
|
4
|
+
import isBinary from '../utils/isbinary.js';
|
|
5
|
+
|
|
6
|
+
const __root = process.cwd();
|
|
7
|
+
|
|
8
|
+
export default async function mount(php, msg) {
|
|
9
|
+
const filesMounted = [];
|
|
10
|
+
|
|
11
|
+
if(msg.patterns) {
|
|
12
|
+
if(typeof msg.patterns != 'object') msg.patterns = [msg.patterns];
|
|
13
|
+
for(const pattern of msg.patterns) {
|
|
14
|
+
await Promise.all(findFiles(pattern).map(async file => {
|
|
15
|
+
const localPath = path.join(__root, file);
|
|
16
|
+
if(!fs.existsSync(localPath)) return;
|
|
17
|
+
const dest = path.posix.join('/project', file);
|
|
18
|
+
const buf = fs.readFileSync(localPath);
|
|
19
|
+
const parentDir = dest.substring(0, dest.lastIndexOf('/'));
|
|
20
|
+
if(parentDir) php.mkdirTree(parentDir);
|
|
21
|
+
php.writeFile(dest, isBinary(buf) ? buf : buf.toString('utf8'));
|
|
22
|
+
filesMounted.push(dest);
|
|
23
|
+
}));
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
return filesMounted;
|
|
28
|
+
}
|