@crawlee/playwright 4.0.0-beta.5 → 4.0.0-beta.50
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -5
- package/internals/adaptive-playwright-crawler.d.ts +48 -56
- package/internals/adaptive-playwright-crawler.d.ts.map +1 -1
- package/internals/adaptive-playwright-crawler.js +310 -206
- package/internals/adaptive-playwright-crawler.js.map +1 -1
- package/internals/enqueue-links/click-elements.d.ts +29 -10
- package/internals/enqueue-links/click-elements.d.ts.map +1 -1
- package/internals/enqueue-links/click-elements.js +42 -18
- package/internals/enqueue-links/click-elements.js.map +1 -1
- package/internals/playwright-crawler.d.ts +55 -51
- package/internals/playwright-crawler.d.ts.map +1 -1
- package/internals/playwright-crawler.js +52 -13
- package/internals/playwright-crawler.js.map +1 -1
- package/internals/playwright-launcher.d.ts +2 -0
- package/internals/playwright-launcher.d.ts.map +1 -1
- package/internals/playwright-launcher.js +1 -1
- package/internals/playwright-launcher.js.map +1 -1
- package/internals/utils/playwright-utils.d.ts +19 -6
- package/internals/utils/playwright-utils.d.ts.map +1 -1
- package/internals/utils/playwright-utils.js +104 -95
- package/internals/utils/playwright-utils.js.map +1 -1
- package/internals/utils/rendering-type-prediction.d.ts +9 -5
- package/internals/utils/rendering-type-prediction.d.ts.map +1 -1
- package/internals/utils/rendering-type-prediction.js +58 -23
- package/internals/utils/rendering-type-prediction.js.map +1 -1
- package/package.json +14 -10
- package/tsconfig.build.tsbuildinfo +0 -1
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { RecoverableState } from '@crawlee/core';
|
|
1
2
|
import LogisticRegression from 'ml-logistic-regression';
|
|
2
3
|
import { Matrix } from 'ml-matrix';
|
|
3
4
|
import stringComparison from 'string-comparison';
|
|
@@ -22,27 +23,58 @@ const mean = (values) => (values.length > 0 ? sum(values) / values.length : unde
|
|
|
22
23
|
* @experimental
|
|
23
24
|
*/
|
|
24
25
|
export class RenderingTypePredictor {
|
|
25
|
-
renderingTypeDetectionResults = new Map();
|
|
26
26
|
detectionRatio;
|
|
27
|
-
|
|
28
|
-
constructor({ detectionRatio }) {
|
|
27
|
+
state;
|
|
28
|
+
constructor({ detectionRatio, persistenceOptions }) {
|
|
29
29
|
this.detectionRatio = detectionRatio;
|
|
30
|
-
this.
|
|
30
|
+
this.state = new RecoverableState({
|
|
31
|
+
defaultState: {
|
|
32
|
+
logreg: new LogisticRegression({ numSteps: 1000, learningRate: 0.05 }),
|
|
33
|
+
detectionResults: new Map(),
|
|
34
|
+
},
|
|
35
|
+
serialize: (state) => JSON.stringify({
|
|
36
|
+
logreg: state.logreg.toJSON(),
|
|
37
|
+
detectionResults: Array.from(state.detectionResults.entries()).map(([renderingType, urlPartsByLabel]) => ({
|
|
38
|
+
renderingType,
|
|
39
|
+
urlPartsByLabel: Array.from(urlPartsByLabel.entries()).map(([label, urlParts]) => ({
|
|
40
|
+
label,
|
|
41
|
+
urlParts,
|
|
42
|
+
})),
|
|
43
|
+
})),
|
|
44
|
+
}),
|
|
45
|
+
deserialize: (serializedState) => {
|
|
46
|
+
const { logreg, detectionResults = [] } = JSON.parse(serializedState);
|
|
47
|
+
return {
|
|
48
|
+
logreg: LogisticRegression.load(logreg),
|
|
49
|
+
detectionResults: new Map(detectionResults.map((serializedItem) => [
|
|
50
|
+
serializedItem.renderingType,
|
|
51
|
+
new Map(serializedItem.urlPartsByLabel.map((item) => [item.label, item.urlParts])),
|
|
52
|
+
])),
|
|
53
|
+
};
|
|
54
|
+
},
|
|
55
|
+
persistStateKey: 'rendering-type-predictor-state',
|
|
56
|
+
persistenceEnabled: true,
|
|
57
|
+
...persistenceOptions,
|
|
58
|
+
});
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* Initialize the predictor by restoring persisted state.
|
|
62
|
+
*/
|
|
63
|
+
async initialize() {
|
|
64
|
+
await this.state.initialize();
|
|
31
65
|
}
|
|
32
66
|
/**
|
|
33
67
|
* Predict the rendering type for a given URL and request label.
|
|
34
68
|
*/
|
|
35
69
|
predict({ url, loadedUrl, label }) {
|
|
36
|
-
|
|
70
|
+
const { logreg } = this.state.currentValue;
|
|
71
|
+
if (logreg.classifiers.length === 0) {
|
|
37
72
|
return { renderingType: 'clientOnly', detectionProbabilityRecommendation: 1 };
|
|
38
73
|
}
|
|
39
74
|
const predictionUrl = new URL(loadedUrl ?? url);
|
|
40
75
|
const urlFeature = new Matrix([this.calculateFeatureVector(urlComponents(predictionUrl), label)]);
|
|
41
|
-
const [prediction] =
|
|
42
|
-
const scores = [
|
|
43
|
-
this.logreg.classifiers[0].testScores(urlFeature),
|
|
44
|
-
this.logreg.classifiers[1].testScores(urlFeature),
|
|
45
|
-
];
|
|
76
|
+
const [prediction] = logreg.predict(urlFeature);
|
|
77
|
+
const scores = [logreg.classifiers[0].testScores(urlFeature), logreg.classifiers[1].testScores(urlFeature)];
|
|
46
78
|
return {
|
|
47
79
|
renderingType: prediction === 1 ? 'static' : 'clientOnly',
|
|
48
80
|
detectionProbabilityRecommendation: Math.abs(scores[0] - scores[1]) < 0.1
|
|
@@ -53,26 +85,29 @@ export class RenderingTypePredictor {
|
|
|
53
85
|
/**
|
|
54
86
|
* Store the rendering type for a given URL and request label. This updates the underlying prediction model, which may be costly.
|
|
55
87
|
*/
|
|
56
|
-
storeResult(
|
|
57
|
-
const
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
88
|
+
storeResult(requests, renderingType) {
|
|
89
|
+
const state = this.state.currentValue;
|
|
90
|
+
for (const { url, loadedUrl, label } of Array.isArray(requests) ? requests : [requests]) {
|
|
91
|
+
const resultUrl = new URL(loadedUrl ?? url);
|
|
92
|
+
if (!state.detectionResults.has(renderingType)) {
|
|
93
|
+
state.detectionResults.set(renderingType, new Map());
|
|
94
|
+
}
|
|
95
|
+
if (!state.detectionResults.get(renderingType).has(label)) {
|
|
96
|
+
state.detectionResults.get(renderingType).set(label, []);
|
|
97
|
+
}
|
|
98
|
+
state.detectionResults.get(renderingType).get(label).push(urlComponents(resultUrl));
|
|
63
99
|
}
|
|
64
|
-
this.renderingTypeDetectionResults.get(renderingType).get(label).push(urlComponents(resultUrl));
|
|
65
100
|
this.retrain();
|
|
66
101
|
}
|
|
67
102
|
resultCount(label) {
|
|
68
|
-
return Array.from(this.
|
|
103
|
+
return Array.from(this.state.currentValue.detectionResults.values())
|
|
69
104
|
.map((results) => results.get(label)?.length ?? 0)
|
|
70
105
|
.reduce((acc, value) => acc + value, 0);
|
|
71
106
|
}
|
|
72
107
|
calculateFeatureVector(url, label) {
|
|
73
108
|
return [
|
|
74
|
-
mean((this.
|
|
75
|
-
mean((this.
|
|
109
|
+
mean((this.state.currentValue.detectionResults.get('static')?.get(label) ?? []).map((otherUrl) => calculateUrlSimilarity(url, otherUrl) ?? 0)) ?? 0,
|
|
110
|
+
mean((this.state.currentValue.detectionResults.get('clientOnly')?.get(label) ?? []).map((otherUrl) => calculateUrlSimilarity(url, otherUrl) ?? 0)) ?? 0,
|
|
76
111
|
];
|
|
77
112
|
}
|
|
78
113
|
retrain() {
|
|
@@ -81,7 +116,7 @@ export class RenderingTypePredictor {
|
|
|
81
116
|
[1, 0],
|
|
82
117
|
];
|
|
83
118
|
const Y = [0, 1];
|
|
84
|
-
for (const [renderingType, urlsByLabel] of this.
|
|
119
|
+
for (const [renderingType, urlsByLabel] of this.state.currentValue.detectionResults.entries()) {
|
|
85
120
|
for (const [label, urls] of urlsByLabel) {
|
|
86
121
|
for (const url of urls) {
|
|
87
122
|
X.push(this.calculateFeatureVector(url, label));
|
|
@@ -89,7 +124,7 @@ export class RenderingTypePredictor {
|
|
|
89
124
|
}
|
|
90
125
|
}
|
|
91
126
|
}
|
|
92
|
-
this.logreg.train(new Matrix(X), Matrix.columnVector(Y));
|
|
127
|
+
this.state.currentValue.logreg.train(new Matrix(X), Matrix.columnVector(Y));
|
|
93
128
|
}
|
|
94
129
|
}
|
|
95
130
|
//# sourceMappingURL=rendering-type-prediction.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"rendering-type-prediction.js","sourceRoot":"","sources":["../../../src/internals/utils/rendering-type-prediction.ts"],"names":[],"mappings":"AACA,OAAO,kBAAkB,MAAM,wBAAwB,CAAC;AACxD,OAAO,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AACnC,OAAO,gBAAgB,MAAM,mBAAmB,CAAC;AAMjD,MAAM,aAAa,GAAG,CAAC,GAAQ,EAAiB,EAAE;IAC9C,OAAO,CAAC,GAAG,CAAC,QAAQ,EAAE,GAAG,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;AACtD,CAAC,CAAC;AAEF,MAAM,sBAAsB,GAAG,CAAC,CAAgB,EAAE,CAAgB,EAAsB,EAAE;IACtF,MAAM,MAAM,GAAa,EAAE,CAAC;IAE5B,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAChB,OAAO,CAAC,CAAC;IACb,CAAC;IAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QACpD,MAAM,CAAC,IAAI,CAAC,gBAAgB,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/F,CAAC;IAED,OAAO,GAAG,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC;AACtD,CAAC,CAAC;AAEF,MAAM,GAAG,GAAG,CAAC,MAAgB,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,CAAC,CAAC;AAC7E,MAAM,IAAI,GAAG,CAAC,MAAgB,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"rendering-type-prediction.js","sourceRoot":"","sources":["../../../src/internals/utils/rendering-type-prediction.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AACjD,OAAO,kBAAkB,MAAM,wBAAwB,CAAC;AACxD,OAAO,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AACnC,OAAO,gBAAgB,MAAM,mBAAmB,CAAC;AAMjD,MAAM,aAAa,GAAG,CAAC,GAAQ,EAAiB,EAAE;IAC9C,OAAO,CAAC,GAAG,CAAC,QAAQ,EAAE,GAAG,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;AACtD,CAAC,CAAC;AAEF,MAAM,sBAAsB,GAAG,CAAC,CAAgB,EAAE,CAAgB,EAAsB,EAAE;IACtF,MAAM,MAAM,GAAa,EAAE,CAAC;IAE5B,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAChB,OAAO,CAAC,CAAC;IACb,CAAC;IAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QACpD,MAAM,CAAC,IAAI,CAAC,gBAAgB,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/F,CAAC;IAED,OAAO,GAAG,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC;AACtD,CAAC,CAAC;AAEF,MAAM,GAAG,GAAG,CAAC,MAAgB,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,CAAC,CAAC;AAC7E,MAAM,IAAI,GAAG,CAAC,MAAgB,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;AAUjG;;;;GAIG;AACH,MAAM,OAAO,sBAAsB;IACvB,cAAc,CAAS;IACvB,KAAK,CAGV;IAEH,YAAY,EAAE,cAAc,EAAE,kBAAkB,EAAiC;QAC7E,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;QACrC,IAAI,CAAC,KAAK,GAAG,IAAI,gBAAgB,CAAC;YAC9B,YAAY,EAAE;gBACV,MAAM,EAAE,IAAI,kBAAkB,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC;gBACtE,gBAAgB,EAAE,IAAI,GAAG,EAA2D;aACvF;YACD,SAAS,EAAE,CAAC,KAAK,EAAE,EAAE,CACjB,IAAI,CAAC,SAAS,CAAC;gBACX,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE;gBAC7B,gBAAgB,EAAE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,CAC9D,CAAC,CAAC,aAAa,EAAE,eAAe,CAAC,EAAE,EAAE,CAAC,CAAC;oBACnC,aAAa;oBACb,eAAe,EAAE,KAAK,CAAC,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,QAAQ,CAAC,EAAE,EAAE,CAAC,CAAC;wBAC/E,KAAK;wBACL,QAAQ;qBACX,CAAC,CAAC;iBACN,CAAC,CACL;aACJ,CAAC;YACN,WAAW,EAAE,CAAC,eAAe,EAAE,EAAE;gBAC7B,MAAM,EAAE,MAAM,EAAE,gBAAgB,GAAG,EAAE,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;gBAEtE,OAAO;oBACH,MAAM,EAAE,kBAAkB,CAAC,IAAI,CAAC,MAAM,CAAC;oBACvC,gBAAgB,EAAE,IAAI,GAAG,CACrB,gBAAgB,CAAC,GAAG,CAAC,CAAC,cAAmB,EAAE,EAAE,CAAC;wBAC1C,cAAc,CAAC,aAAa;wBAC5B,IAAI,GAAG,CAAC,cAAc,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,IAAS,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;qBAC1F,CAAC,CACL;iBACJ,CAAC;YACN,CAAC;YACD,eAAe,EAAE,gCAAgC;YACjD,kBAAkB,EAAE,IAAI;YACxB,GAAG,kBAAkB;SACxB,CAAC,CAAC;IACP,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU;QACZ,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,CAAC;IAClC,CAAC;IAED;;OAEG;IACI,OAAO,CAAC,EAAE,GAAG,EAAE,SAAS,EAAE,KAAK,EAAW;QAI7C,MAAM,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC;QAC3C,IAAI,MAAM,CAAC,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAClC,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,kCAAkC,EAAE,CAAC,EAAE,CAAC;QAClF,CAAC;QAED,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,SAAS,IAAI,GAAG,CAAC,CAAC;QAEhD,MAAM,UAAU,GAAG,IAAI,MAAM,CAAC,CAAC,IAAI,CAAC,sBAAsB,CAAC,aAAa,CAAC,aAAa,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;QAClG,MAAM,CAAC,UAAU,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;QAChD,MAAM,MAAM,GAAG,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC,CAAC;QAE5G,OAAO;YACH,aAAa,EAAE,UAAU,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,YAAY;YACzD,kCAAkC,EAC9B,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG;gBACjC,CAAC,CAAC,CAAC;gBACH,CAAC,CAAC,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;SAC3E,CAAC;IACN,CAAC;IAED;;OAEG;IACI,WAAW,CAAC,QAA6B,EAAE,aAA4B;QAC1E,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC;QAEtC,KAAK,MAAM,EAAE,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC;YACtF,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,SAAS,IAAI,GAAG,CAAC,CAAC;YAE5C,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,GAAG,CAAC,aAAa,CAAC,EAAE,CAAC;gBAC7C,KAAK,CAAC,gBAAgB,CAAC,GAAG,CAAC,aAAa,EAAE,IAAI,GAAG,EAAE,CAAC,CAAC;YACzD,CAAC;YAED,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,GAAG,CAAC,aAAa,CAAE,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;gBACzD,KAAK,CAAC,gBAAgB,CAAC,GAAG,CAAC,aAAa,CAAE,CAAC,GAAG,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;YAC9D,CAAC;YAED,KAAK,CAAC,gBAAgB,CAAC,GAAG,CAAC,aAAa,CAAE,CAAC,GAAG,CAAC,KAAK,CAAE,CAAC,IAAI,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC,CAAC;QAC1F,CAAC;QAED,IAAI,CAAC,OAAO,EAAE,CAAC;IACnB,CAAC;IAEO,WAAW,CAAC,KAAyB;QACzC,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,gBAAgB,CAAC,MAAM,EAAE,CAAC;aAC/D,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,IAAI,CAAC,CAAC;aACjD,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,EAAE,CAAC,CAAC,CAAC;IAChD,CAAC;IAES,sBAAsB,CAAC,GAAkB,EAAE,KAAyB;QAC1E,OAAO;YACH,IAAI,CACA,CAAC,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,gBAAgB,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CAC1E,CAAC,QAAQ,EAAE,EAAE,CAAC,sBAAsB,CAAC,GAAG,EAAE,QAAQ,CAAC,IAAI,CAAC,CAC3D,CACJ,IAAI,CAAC;YACN,IAAI,CACA,CAAC,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,gBAAgB,CAAC,GAAG,CAAC,YAAY,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CAC9E,CAAC,QAAQ,EAAE,EAAE,CAAC,sBAAsB,CAAC,GAAG,EAAE,QAAQ,CAAC,IAAI,CAAC,CAC3D,CACJ,IAAI,CAAC;SACT,CAAC;IACN,CAAC;IAES,OAAO;QACb,MAAM,CAAC,GAAoB;YACvB,CAAC,CAAC,EAAE,CAAC,CAAC;YACN,CAAC,CAAC,EAAE,CAAC,CAAC;SACT,CAAC;QACF,MAAM,CAAC,GAAa,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAE3B,KAAK,MAAM,CAAC,aAAa,EAAE,WAAW,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,gBAAgB,CAAC,OAAO,EAAE,EAAE,CAAC;YAC5F,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,WAAW,EAAE,CAAC;gBACtC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;oBACrB,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,sBAAsB,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC;oBAChD,CAAC,CAAC,IAAI,CAAC,aAAa,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC/C,CAAC;YACL,CAAC;QACL,CAAC;QAED,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC;IAChF,CAAC;CACJ"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/playwright",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.50",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.0.0"
|
|
@@ -38,7 +38,7 @@
|
|
|
38
38
|
},
|
|
39
39
|
"homepage": "https://crawlee.dev",
|
|
40
40
|
"scripts": {
|
|
41
|
-
"build": "
|
|
41
|
+
"build": "pnpm clean && pnpm compile && pnpm copy",
|
|
42
42
|
"clean": "rimraf ./dist",
|
|
43
43
|
"compile": "tsc -p tsconfig.build.json",
|
|
44
44
|
"copy": "tsx ../../scripts/copy.ts"
|
|
@@ -48,17 +48,17 @@
|
|
|
48
48
|
},
|
|
49
49
|
"dependencies": {
|
|
50
50
|
"@apify/datastructures": "^2.0.3",
|
|
51
|
-
"@apify/log": "^2.5.18",
|
|
52
51
|
"@apify/timeout": "^0.3.2",
|
|
53
|
-
"@crawlee/
|
|
54
|
-
"@crawlee/browser
|
|
55
|
-
"@crawlee/
|
|
56
|
-
"@crawlee/
|
|
57
|
-
"@crawlee/
|
|
52
|
+
"@crawlee/basic": "4.0.0-beta.50",
|
|
53
|
+
"@crawlee/browser": "4.0.0-beta.50",
|
|
54
|
+
"@crawlee/browser-pool": "4.0.0-beta.50",
|
|
55
|
+
"@crawlee/cheerio": "4.0.0-beta.50",
|
|
56
|
+
"@crawlee/core": "4.0.0-beta.50",
|
|
57
|
+
"@crawlee/types": "4.0.0-beta.50",
|
|
58
|
+
"@crawlee/utils": "4.0.0-beta.50",
|
|
58
59
|
"cheerio": "^1.0.0",
|
|
59
60
|
"idcac-playwright": "^0.1.3",
|
|
60
61
|
"jquery": "^3.7.1",
|
|
61
|
-
"lodash.isequal": "^4.5.0",
|
|
62
62
|
"ml-logistic-regression": "^2.0.0",
|
|
63
63
|
"ml-matrix": "^6.12.1",
|
|
64
64
|
"ow": "^2.0.0",
|
|
@@ -66,9 +66,13 @@
|
|
|
66
66
|
"tslib": "^2.8.1"
|
|
67
67
|
},
|
|
68
68
|
"peerDependencies": {
|
|
69
|
+
"idcac-playwright": "^0.2.0",
|
|
69
70
|
"playwright": "*"
|
|
70
71
|
},
|
|
71
72
|
"peerDependenciesMeta": {
|
|
73
|
+
"idcac-playwright": {
|
|
74
|
+
"optional": true
|
|
75
|
+
},
|
|
72
76
|
"playwright": {
|
|
73
77
|
"optional": true
|
|
74
78
|
}
|
|
@@ -80,5 +84,5 @@
|
|
|
80
84
|
}
|
|
81
85
|
}
|
|
82
86
|
},
|
|
83
|
-
"gitHead": "
|
|
87
|
+
"gitHead": "cdd082a1e9d106e6ea46fbe21928cc2f1978f4f7"
|
|
84
88
|
}
|