ag-webscrape 0.0.6 → 0.0.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/WebScraper.d.ts.map +1 -1
- package/dist/WebScraper.js +4 -1
- package/dist/WebScraper.js.map +1 -1
- package/dist/helpers/dom.d.ts.map +1 -1
- package/dist/helpers/dom.js +10 -28
- package/dist/helpers/dom.js.map +1 -1
- package/dist/test.js +1 -0
- package/dist/test.js.map +1 -1
- package/package.json +10 -10
package/dist/WebScraper.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"WebScraper.d.ts","sourceRoot":"","sources":["../src/WebScraper.ts"],"names":[],"mappings":"AAKA,MAAM,WAAW,eAAe;IAC9B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,cAAc;IAC7B,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,OAAO,GAAG,QAAQ,CAAC;IAC3B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,qBAAa,UAAU;IACrB,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,cAAc,CAAkB;gBAE5B,OAAO,GAAE,eAAoB;YAc3B,aAAa;YAiDb,mBAAmB;IAsEjC,OAAO,CAAC,WAAW;IAmBb,MAAM,CACV,GAAG,EAAE,MAAM,EACX,OAAO,GAAE,eAAoB,GAC5B,OAAO,CAAC,cAAc,CAAC;
|
|
1
|
+
{"version":3,"file":"WebScraper.d.ts","sourceRoot":"","sources":["../src/WebScraper.ts"],"names":[],"mappings":"AAKA,MAAM,WAAW,eAAe;IAC9B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,cAAc;IAC7B,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,OAAO,GAAG,QAAQ,CAAC;IAC3B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,qBAAa,UAAU;IACrB,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,cAAc,CAAkB;gBAE5B,OAAO,GAAE,eAAoB;YAc3B,aAAa;YAiDb,mBAAmB;IAsEjC,OAAO,CAAC,WAAW;IAmBb,MAAM,CACV,GAAG,EAAE,MAAM,EACX,OAAO,GAAE,eAAoB,GAC5B,OAAO,CAAC,cAAc,CAAC;IAqDpB,cAAc,CAClB,IAAI,EAAE,MAAM,EAAE,EACd,OAAO,GAAE,eAAoB,GAC5B,OAAO,CAAC,cAAc,EAAE,CAAC;IAwBtB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAG/B"}
|
package/dist/WebScraper.js
CHANGED
|
@@ -133,16 +133,19 @@ class WebScraper {
|
|
|
133
133
|
}
|
|
134
134
|
try {
|
|
135
135
|
const result = await this.scrapeWithpuppeteer(url, mergedOptions);
|
|
136
|
+
(0, log_1.info)(`Puppeteer scrape successful for ${url}.`, JSON.stringify(result, null, 2));
|
|
136
137
|
return result;
|
|
137
138
|
}
|
|
138
139
|
catch (error) {
|
|
139
140
|
const puppeteerError = error instanceof Error ? error : new Error('Unknown puppeteer error');
|
|
141
|
+
const m = `Both methods failed. Fetch: ${lastError?.message || 'Unknown'}. puppeteer: ${puppeteerError.message}. err=${error.message}`;
|
|
142
|
+
(0, log_1.warn)(m);
|
|
140
143
|
return {
|
|
141
144
|
url,
|
|
142
145
|
html: '',
|
|
143
146
|
status: 0,
|
|
144
147
|
method: 'visual',
|
|
145
|
-
error:
|
|
148
|
+
error: m,
|
|
146
149
|
};
|
|
147
150
|
}
|
|
148
151
|
}
|
package/dist/WebScraper.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"WebScraper.js","sourceRoot":"","sources":["../src/WebScraper.ts"],"names":[],"mappings":";;;AACA,
|
|
1
|
+
{"version":3,"file":"WebScraper.js","sourceRoot":"","sources":["../src/WebScraper.ts"],"names":[],"mappings":";;;AACA,2DAA+D;AAE/D,uCAAuD;AAqBvD,MAAa,UAAU;IAIrB,YAAY,UAA2B,EAAE;QACvC,IAAI,CAAC,SAAS;YACZ,iHAAiH,CAAC;QACpH,IAAI,CAAC,cAAc,GAAG;YACpB,OAAO,EAAE,KAAK;YACd,OAAO,EAAE,CAAC;YACV,cAAc,EAAE,IAAI;YACpB,GAAG,OAAO;SACX,CAAC;IACJ,CAAC;IAKO,KAAK,CAAC,aAAa,CACzB,GAAW,EACX,OAAwB;QAExB,MAAM,OAAO,GAAG;YACd,YAAY,EAAE,OAAO,CAAC,SAAS,IAAI,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE;YAC5D,MAAM,EACJ,4EAA4E;YAC9E,iBAAiB,EAAE,gBAAgB;YACnC,iBAAiB,EAAE,eAAe;YAClC,UAAU,EAAE,YAAY;YACxB,2BAA2B,EAAE,GAAG;YAChC,GAAG,OAAO,CAAC,OAAO;SACnB,CAAC;QAEF,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,SAAS,GAAG,UAAU,CAC1B,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EACxB,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,cAAc,CAAC,OAAQ,CAChD,CAAC;QAEF,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;gBAChC,OAAO;gBACP,MAAM,EAAE,UAAU,CAAC,MAAM;gBACzB,QAAQ,EAAE,QAAQ;aACnB,CAAC,CAAC;YAEH,YAAY,CAAC,SAAS,CAAC,CAAC;YAExB,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YAEnC,OAAO;gBACL,GAAG;gBACH,IAAI;gBACJ,MAAM,EAAE,QAAQ,CAAC,MAAM;gBACvB,MAAM,EAAE,OAAO;gBACf,UAAU,EAAE,QAAQ,CAAC,UAAU;gBAC/B,QAAQ,EAAE,QAAQ,CAAC,GAAG;aACvB,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,YAAY,CAAC,SAAS,CAAC,CAAC;YACxB,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAKO,KAAK,CAAC,mBAAmB,CAC/B,GAAW,EACX,OAAwB;QAExB,IAAI,IAAI,GAAG,EAAE,CAAC;QACd,IAAI,MAAM,GAAG,GAAG,CAAC;QACjB,IAAI,KAAyB,CAAC;QAC9B,IAAI,QAAQ,GAAG,GAAG,CAAC;QAEnB,IAAI,CAAC;YAEH,MAAM,UAAU,GAAG,MAAM,IAAA,cAAQ,EAAC,GAAG,EAAE;gBACrC,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,cAAc,CAAC,OAAO;gBACvD,iBAAiB,EAAE,OAAO,CAAC,eAAe;aAC3C,CAAC,CAAC;YAGH,IAAI,GAAG,UAAU,CAAC,SAAS,CAAC;YAC5B,QAAQ,GAAG,GAAG,CAAC;YAGf,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC3B,KAAK,GAAG,qCAAqC,CAAC;gBAC9C,MAAM,GAAG,GAAG,CAAC;YACf,CAAC;YAED,OAAO;gBACL,GAAG;gBACH,IAAI;gBACJ,MAAM;gBACN,MAAM,EAAE,QAAQ;gBAChB,KAAK;gBACL,QAAQ;aACT,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,YAAY,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC;YAG1E,IAAI,YAAY,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;gBACrC,MAAM,GAAG,GAAG,CAAC;YACf,CAAC;iBAAM,IACL,YAAY,CAAC,QAAQ,CAAC,KAAK,CAAC;gBAC5B,YAAY,CAAC,QAAQ,CAAC,WAAW,CAAC,EAClC,CAAC;gBACD,MAAM,GAAG,GAAG,CAAC;YACf,CAAC;iBAAM,IACL,YAAY,CAAC,QAAQ,CAAC,KAAK,CAAC;gBAC5B,YAAY,CAAC,QAAQ,CAAC,WAAW,CAAC,EAClC,CAAC;gBACD,MAAM,GAAG,GAAG,CAAC;YACf,CAAC;iBAAM,IAAI,YAAY,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;gBACxC,MAAM,GAAG,GAAG,CAAC;YACf,CAAC;iBAAM,CAAC;gBACN,MAAM,GAAG,CAAC,CAAC;YACb,CAAC;YAED,OAAO;gBACL,GAAG;gBACH,IAAI,EAAE,EAAE;gBACR,MAAM;gBACN,MAAM,EAAE,QAAQ;gBAChB,KAAK,EAAE,YAAY;gBACnB,QAAQ;aACT,CAAC;QACJ,CAAC;IACH,CAAC;IAKO,WAAW,CAAC,IAAY;QAC9B,MAAM,aAAa,GAAG;YACpB,4BAA4B;YAC5B,4BAA4B;YAC5B,4BAA4B;YAC5B,gCAAgC;YAChC,gCAAgC;YAChC,mCAAmC;YACnC,sBAAsB;YACtB,sBAAsB;YACtB,sBAAsB;SACvB,CAAC;QAEF,OAAO,aAAa,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IAC7D,CAAC;IAKD,KAAK,CAAC,MAAM,CACV,GAAW,EACX,UAA2B,EAAE;QAE7B,MAAM,aAAa,GAAG,EAAE,GAAG,IAAI,CAAC,cAAc,EAAE,GAAG,OAAO,EAAE,CAAC;QAC7D,IAAI,SAAS,GAAiB,IAAI,CAAC;QAGnC,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,GAAG,EAAE,aAAa,CAAC,CAAC;YAG5D,IAAI,MAAM,CAAC,MAAM,IAAI,GAAG,IAAI,MAAM,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;gBAChD,OAAO,MAAM,CAAC;YAChB,CAAC;YAGD,IAAA,UAAI,EACF,qDAAqD,GAAG,8BAA8B,EACtF,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAChC,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,SAAS;gBACP,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;YACpE,IAAA,UAAI,EACF,2BAA2B,GAAG,KAAK,SAAS,CAAC,OAAO,8BAA8B,CACnF,CAAC;QACJ,CAAC;QAGD,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,GAAG,EAAE,aAAa,CAAC,CAAC;YAClE,IAAA,UAAI,EACF,mCAAmC,GAAG,GAAG,EACzC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAChC,CAAC;YACF,OAAO,MAAM,CAAC;QAChB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,cAAc,GAClB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;YAExE,MAAM,CAAC,GAAG,+BAA+B,SAAS,EAAE,OAAO,IAAI,SAAS,gBAAgB,cAAc,CAAC,OAAO,SAAU,KAAe,CAAC,OAAO,EAAE,CAAC;YAClJ,IAAA,UAAI,EAAC,CAAC,CAAC,CAAC;YACR,OAAO;gBACL,GAAG;gBACH,IAAI,EAAE,EAAE;gBACR,MAAM,EAAE,CAAC;gBACT,MAAM,EAAE,QAAQ;gBAChB,KAAK,EAAE,CAAC;aACT,CAAC;QACJ,CAAC;IACH,CAAC;IAKD,KAAK,CAAC,cAAc,CAClB,IAAc,EACd,UAA2B,EAAE;QAE7B,MAAM,OAAO,GAAqB,EAAE,CAAC;QAErC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;gBAC/C,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,IAAI,CAAC;oBACX,GAAG;oBACH,IAAI,EAAE,EAAE;oBACR,MAAM,EAAE,CAAC;oBACT,MAAM,EAAE,OAAO;oBACf,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe;iBAChE,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAKD,KAAK,CAAC,OAAO;QACX,MAAM,IAAA,kBAAY,GAAE,CAAC;IACvB,CAAC;CACF;AAlPD,gCAkPC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dom.d.ts","sourceRoot":"","sources":["../../src/helpers/dom.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAiCpD,eAAO,MAAM,aAAa,
|
|
1
|
+
{"version":3,"file":"dom.d.ts","sourceRoot":"","sources":["../../src/helpers/dom.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAiCpD,eAAO,MAAM,aAAa,qBAgCzB,CAAC;AAEF,eAAO,MAAM,YAAY,qBAUxB,CAAC;AAEF,eAAO,MAAM,QAAQ,GACnB,KAAK,MAAM,GAAG,GAAG,EACjB,MAAM;IAEJ,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B,KACA,OAAO,CAAC,WAAW,CAwErB,CAAC"}
|
package/dist/helpers/dom.js
CHANGED
|
@@ -38,39 +38,21 @@ const launchBrowser = async () => {
|
|
|
38
38
|
height: 1920,
|
|
39
39
|
width: 1080,
|
|
40
40
|
},
|
|
41
|
-
args: chromium_1.default.args,
|
|
42
41
|
headless: process.env.HEADLESS === 'false' ? false : true,
|
|
43
42
|
ignoreHTTPSErrors: true,
|
|
44
43
|
devtools: false,
|
|
45
44
|
executablePath,
|
|
45
|
+
args: [
|
|
46
|
+
'--no-sandbox',
|
|
47
|
+
'--disable-setuid-sandbox',
|
|
48
|
+
'--disable-dev-shm-usage',
|
|
49
|
+
'--disable-accelerated-2d-canvas',
|
|
50
|
+
'--no-first-run',
|
|
51
|
+
'--no-zygote',
|
|
52
|
+
'--single-process',
|
|
53
|
+
'--disable-gpu',
|
|
54
|
+
],
|
|
46
55
|
};
|
|
47
|
-
if (!opt.args) {
|
|
48
|
-
opt.args = [];
|
|
49
|
-
}
|
|
50
|
-
opt.args.push('--disable-features=AudioServiceOutOfProcess');
|
|
51
|
-
opt.args.push('--disable-features=AudioServiceOutOfProcessKillAtHang');
|
|
52
|
-
opt.args.push('--disable-software-rasterizer');
|
|
53
|
-
opt.args.push('--disable-gpu');
|
|
54
|
-
opt.args.push('--disable-dev-shm-usage');
|
|
55
|
-
opt.args.push('--disable-setuid-sandbox');
|
|
56
|
-
opt.args.push('--disable-web-security');
|
|
57
|
-
opt.args.push('--disable-features=site-per-process');
|
|
58
|
-
opt.args.push('--disable-features=VizDisplayCompositor');
|
|
59
|
-
opt.args.push('--disable-extensions');
|
|
60
|
-
opt.args.push('--disable-plugins');
|
|
61
|
-
opt.args.push('--disable-default-apps');
|
|
62
|
-
opt.args.push('--disable-background-timer-throttling');
|
|
63
|
-
opt.args.push('--disable-backgrounding-occluded-windows');
|
|
64
|
-
opt.args.push('--disable-renderer-backgrounding');
|
|
65
|
-
opt.args.push('--disable-hang-monitor');
|
|
66
|
-
opt.args.push('--disable-ipc-flooding-protection');
|
|
67
|
-
opt.args.push('--disable-prompt-on-repost');
|
|
68
|
-
opt.args.push('--disable-client-side-phishing-detection');
|
|
69
|
-
opt.args.push('--disable-component-update');
|
|
70
|
-
opt.args.push('--disable-domain-reliability');
|
|
71
|
-
opt.args.push('--disable-features=TranslateUI');
|
|
72
|
-
opt.args.push('--disable-features=BlinkGenPropertyTrees');
|
|
73
|
-
opt.args.push('--enable-features=NetworkService,NetworkServiceInProcess');
|
|
74
56
|
(0, log_1.trace)('launch browser, opt=', opt);
|
|
75
57
|
try {
|
|
76
58
|
if (browser?.close) {
|
package/dist/helpers/dom.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dom.js","sourceRoot":"","sources":["../../src/helpers/dom.ts"],"names":[],"mappings":";;;;;;AACA,mEAA2C;AAC3C,2DAA8E;AAC9E,2BAAgC;AAEhC,uDAAyC;AAEzC,mDAAwC;AAExC,IAAI,OAA4B,CAAC;AAEjC,MAAM,mBAAmB,GAAG,KAAK,IAAI,EAAE;IACrC,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;IAElC,IAAI,QAAQ,KAAK,OAAO,EAAE,CAAC;QAEzB,MAAM,aAAa,GAAG;YACpB,4DAA4D;YAC5D,kEAAkE;YAClE,6DAA6D;YAC7D,mEAAmE;SACpE,CAAC;QAEF,KAAK,MAAM,IAAI,IAAI,aAAa,EAAE,CAAC;YACjC,IAAI,CAAC;gBACH,IAAA,eAAU,EAAC,IAAI,CAAC,CAAC;gBACjB,OAAO,IAAI,CAAC;YACd,CAAC;YAAC,MAAM,CAAC;YAET,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,GAAG,GAAG,MAAM,kBAAQ,CAAC,cAAc,EAAE,CAAC;IAC5C,OAAO,GAAG,CAAC;AACb,CAAC,CAAC;AAEK,MAAM,aAAa,GAAG,KAAK,IAAI,EAAE;IACtC,MAAM,cAAc,GAAG,MAAM,mBAAmB,EAAE,CAAC;IACnD,MAAM,GAAG,GAAG;QACV,eAAe,EAAE;YACf,MAAM,EAAE,IAAI;YACZ,KAAK,EAAE,IAAI;SACZ;QACD,
|
|
1
|
+
{"version":3,"file":"dom.js","sourceRoot":"","sources":["../../src/helpers/dom.ts"],"names":[],"mappings":";;;;;;AACA,mEAA2C;AAC3C,2DAA8E;AAC9E,2BAAgC;AAEhC,uDAAyC;AAEzC,mDAAwC;AAExC,IAAI,OAA4B,CAAC;AAEjC,MAAM,mBAAmB,GAAG,KAAK,IAAI,EAAE;IACrC,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;IAElC,IAAI,QAAQ,KAAK,OAAO,EAAE,CAAC;QAEzB,MAAM,aAAa,GAAG;YACpB,4DAA4D;YAC5D,kEAAkE;YAClE,6DAA6D;YAC7D,mEAAmE;SACpE,CAAC;QAEF,KAAK,MAAM,IAAI,IAAI,aAAa,EAAE,CAAC;YACjC,IAAI,CAAC;gBACH,IAAA,eAAU,EAAC,IAAI,CAAC,CAAC;gBACjB,OAAO,IAAI,CAAC;YACd,CAAC;YAAC,MAAM,CAAC;YAET,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,GAAG,GAAG,MAAM,kBAAQ,CAAC,cAAc,EAAE,CAAC;IAC5C,OAAO,GAAG,CAAC;AACb,CAAC,CAAC;AAEK,MAAM,aAAa,GAAG,KAAK,IAAI,EAAE;IACtC,MAAM,cAAc,GAAG,MAAM,mBAAmB,EAAE,CAAC;IACnD,MAAM,GAAG,GAAG;QACV,eAAe,EAAE;YACf,MAAM,EAAE,IAAI;YACZ,KAAK,EAAE,IAAI;SACZ;QACD,QAAQ,EAAE,OAAO,CAAC,GAAG,CAAC,QAAQ,KAAK,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI;QACzD,iBAAiB,EAAE,IAAI;QACvB,QAAQ,EAAE,KAAK;QACf,cAAc;QACd,IAAI,EAAE;YACJ,cAAc;YACd,0BAA0B;YAC1B,yBAAyB;YACzB,iCAAiC;YACjC,gBAAgB;YAChB,aAAa;YACb,kBAAkB;YAClB,eAAe;SAChB;KACF,CAAC;IAEF,IAAA,WAAK,EAAC,sBAAsB,EAAE,GAAG,CAAC,CAAC;IACnC,IAAI,CAAC;QACH,IAAI,OAAO,EAAE,KAAK,EAAE,CAAC;YACnB,MAAM,OAAO,EAAE,KAAK,EAAE,CAAC;QACzB,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;IAET,CAAC;IACD,OAAO,GAAG,CAAC,MAAM,IAAA,uBAAM,EAAC,GAAG,CAAC,CAAuB,CAAC;AACtD,CAAC,CAAC;AAhCW,QAAA,aAAa,iBAgCxB;AAEK,MAAM,YAAY,GAAG,KAAK,IAAI,EAAE;IACrC,IAAI,CAAC;QACH,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,OAAO;QACT,CAAC;QAED,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;IACxB,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,IAAA,UAAI,EAAC,wBAAwB,EAAE,CAAC,CAAC,CAAC;IACpC,CAAC;AACH,CAAC,CAAC;AAVW,QAAA,YAAY,gBAUvB;AAEK,MAAM,QAAQ,GAAG,KAAK,EAC3B,GAAiB,EACjB,GAIC,EACqB,EAAE;IACxB,IAAI,UAAU,GAAG,KAAK,CAAC;IACvB,GAAG,CAAC;QACF,IAAI,CAAC;YACH,IAAI,CAAC,OAAO,EAAE,CAAC;gBACb,MAAM,IAAA,qBAAa,GAAE,CAAC;YACxB,CAAC;YAED,IAAA,WAAK,EAAC,aAAa,GAAG,GAAG,CAAC,CAAC;YAC3B,MAAM,IAAI,GAAG,MAAM,OAAQ,CAAC,OAAO,EAAE,CAAC;YACtC,IAAI,CAAC,GAAG,GAAG,EAAE,OAAO,IAAI,IAAI,CAAC;YAC7B,IAAI,UAAU,EAAE,CAAC;gBACf,CAAC,IAAI,IAAI,CAAC;YACZ,CAAC;YAED,MAAM,IAAI,GAAG,OAAO,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;YAE5D,IAAI,CAAC,GAAG,EAAE,iBAAiB,EAAE,CAAC;gBAE5B,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;oBACpB,SAAS,EAAE,CAAC,MAAM,EAAE,kBAAkB,CAAC;oBACvC,OAAO,EAAE,CAAC;iBACX,CAAC,CAAC;YACL,CAAC;iBAAM,CAAC;gBACN,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;oBACpB,SAAS,EAAE,CAAC,MAAM,CAAC;oBACnB,OAAO,EAAE,CAAC;iBACX,CAAC,CAAC;gBACH,MAAM,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,iBAAiB,EAAE;oBAChD,OAAO,EAAE,CAAC;oBACV,OAAO,EAAE,IAAI;iBACd,CAAC,CAAC;YACL,CAAC;YAED,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;YACrC,MAAM,GAAG,GAAG,IAAA,wBAAK,EAAC,OAAO,CAAC,CAAC;YAE3B,GAAG,CAAC,gBAAgB,CAAC,kBAAkB,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;YACrE,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;YACnB,UAAU,GAAG,KAAK,CAAC;YACnB,OAAO,GAAG,CAAC;QACb,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,GAAG,GAAY,CAAC;YACvB,IAAI,UAAU,EAAE,CAAC;gBACf,IAAA,WAAK,EAAC,qBAAqB,EAAE,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;gBAChD,MAAM,CAAC,CAAC;YACV,CAAC;YAGD,IACE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,kBAAkB,CAAC;gBACzC,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,YAAY,CAAC;gBACnC,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,oBAAoB,CAAC;gBAC3C,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,+BAA+B,CAAC;gBACtD,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,gBAAgB,CAAC;gBACvC,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,eAAe,CAAC,EACtC,CAAC;gBACD,IAAI,CAAC;oBACH,IAAA,UAAI,EAAC,QAAQ,EAAE,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;oBAClC,MAAM,IAAA,qBAAa,GAAE,CAAC;oBACtB,UAAU,GAAG,IAAI,CAAC;gBACpB,CAAC;gBAAC,OAAO,EAAE,EAAE,CAAC;oBACZ,IAAA,WAAK,EAAC,4BAA4B,EAAE,EAAE,CAAC,CAAC;oBACxC,MAAM,EAAE,CAAC;gBACX,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,IAAA,WAAK,EAAC,gBAAgB,CAAC,EAAE,CAAC,CAAC;gBAC3B,MAAM,CAAC,CAAC;YACV,CAAC;QACH,CAAC;IACH,CAAC,QAAQ,UAAU,EAAE;IACrB,MAAM,IAAI,KAAK,CAAC,iBAAiB,CAAC,CAAC;AACrC,CAAC,CAAC;AA/EW,QAAA,QAAQ,YA+EnB"}
|
package/dist/test.js
CHANGED
|
@@ -5,6 +5,7 @@ async function main() {
|
|
|
5
5
|
const ws = new WebScraper_1.WebScraper();
|
|
6
6
|
const result = await ws.scrape('https://bothsidesofthetable.com');
|
|
7
7
|
console.log('result=', JSON.stringify(result, null, 2));
|
|
8
|
+
await ws.dispose();
|
|
8
9
|
}
|
|
9
10
|
void main();
|
|
10
11
|
//# sourceMappingURL=test.js.map
|
package/dist/test.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"test.js","sourceRoot":"","sources":["../src/test.ts"],"names":[],"mappings":";;AAAA,6CAA0C;AAE1C,KAAK,UAAU,IAAI;IACjB,MAAM,EAAE,GAAG,IAAI,uBAAU,EAAE,CAAC;IAC5B,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,MAAM,CAAC,iCAAiC,CAAC,CAAC;IAClE,OAAO,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"test.js","sourceRoot":"","sources":["../src/test.ts"],"names":[],"mappings":";;AAAA,6CAA0C;AAE1C,KAAK,UAAU,IAAI;IACjB,MAAM,EAAE,GAAG,IAAI,uBAAU,EAAE,CAAC;IAC5B,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,MAAM,CAAC,iCAAiC,CAAC,CAAC;IAClE,OAAO,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IACxD,MAAM,EAAE,CAAC,OAAO,EAAE,CAAC;AACrB,CAAC;AAED,KAAK,IAAI,EAAE,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "ag-webscrape",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.8",
|
|
4
4
|
"author": "admin@gec.dev",
|
|
5
5
|
"description": "TypeScript web scraper with Playwright fallback for anti-scraping protection",
|
|
6
6
|
"main": "dist/index.js",
|
|
@@ -14,18 +14,18 @@
|
|
|
14
14
|
],
|
|
15
15
|
"license": "MIT",
|
|
16
16
|
"dependencies": {
|
|
17
|
-
"@sparticuz/chromium": "138.0.1",
|
|
18
|
-
"ag-common": "0.0.752",
|
|
17
|
+
"@sparticuz/chromium": "^138.0.1",
|
|
18
|
+
"ag-common": "^0.0.752",
|
|
19
19
|
"node-html-parser": "^7.0.1",
|
|
20
|
-
"puppeteer": "^24.
|
|
21
|
-
"puppeteer-core": "^24.
|
|
20
|
+
"puppeteer": "^24.14.0",
|
|
21
|
+
"puppeteer-core": "^24.14.0"
|
|
22
22
|
},
|
|
23
23
|
"devDependencies": {
|
|
24
|
-
"@types/node": "24.0.
|
|
25
|
-
"eslint": "9.31.0",
|
|
26
|
-
"eslint-config-e7npm": "0.1.23",
|
|
27
|
-
"tsx": "4.20.3",
|
|
28
|
-
"typescript": "5.8.3"
|
|
24
|
+
"@types/node": "^24.0.15",
|
|
25
|
+
"eslint": "^9.31.0",
|
|
26
|
+
"eslint-config-e7npm": "^0.1.23",
|
|
27
|
+
"tsx": "^4.20.3",
|
|
28
|
+
"typescript": "^5.8.3"
|
|
29
29
|
},
|
|
30
30
|
"files": [
|
|
31
31
|
"dist/**/*",
|