@crawlee/impit-client 4.0.0-beta.12 → 4.0.0-beta.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.d.ts +3 -3
- package/index.d.ts.map +1 -1
- package/index.js +8 -41
- package/index.js.map +1 -1
- package/package.json +2 -2
- package/tsconfig.build.tsbuildinfo +1 -1
package/index.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type
|
|
1
|
+
import { type BaseHttpClient, type HttpRequest, type ResponseTypes } from '@crawlee/core';
|
|
2
2
|
import type { ImpitOptions } from 'impit';
|
|
3
3
|
export declare const Browser: {
|
|
4
4
|
readonly Chrome: "chrome";
|
|
@@ -38,11 +38,11 @@ export declare class ImpitHttpClient implements BaseHttpClient {
|
|
|
38
38
|
/**
|
|
39
39
|
* @inheritDoc
|
|
40
40
|
*/
|
|
41
|
-
sendRequest<TResponseType extends keyof ResponseTypes>(request: HttpRequest<TResponseType>): Promise<
|
|
41
|
+
sendRequest<TResponseType extends keyof ResponseTypes>(request: HttpRequest<TResponseType>): Promise<Response>;
|
|
42
42
|
private getStreamWithProgress;
|
|
43
43
|
/**
|
|
44
44
|
* @inheritDoc
|
|
45
45
|
*/
|
|
46
|
-
stream(request: HttpRequest): Promise<
|
|
46
|
+
stream(request: HttpRequest): Promise<Response>;
|
|
47
47
|
}
|
|
48
48
|
//# sourceMappingURL=index.d.ts.map
|
package/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,KAAK,cAAc,EAAE,KAAK,WAAW,EAAE,KAAK,aAAa,EAAmB,MAAM,eAAe,CAAC;AAC3G,OAAO,KAAK,EAAc,YAAY,EAA8B,MAAM,OAAO,CAAC;AAKlF,eAAO,MAAM,OAAO;;;CAGV,CAAC;AAOX;;GAEG;AACH,qBAAa,eAAgB,YAAW,cAAc;IAClD,OAAO,CAAC,YAAY,CAAe;IACnC,OAAO,CAAC,YAAY,CAAS;IAC7B,OAAO,CAAC,eAAe,CAAU;IAEjC;;;;;OAKG;IACH,OAAO,CAAC,WAAW,CAAoD;IAEvE,OAAO,CAAC,SAAS;gBAaL,OAAO,CAAC,EAAE,IAAI,CAAC,YAAY,EAAE,UAAU,CAAC,GAAG;QAAE,YAAY,CAAC,EAAE,MAAM,CAAA;KAAE;IAOhF;;;;OAIG;IACH,OAAO,CAAC,WAAW;IAsBnB,OAAO,CAAC,aAAa;IAarB;;;;OAIG;YACW,WAAW;IAmDzB;;OAEG;IACG,WAAW,CAAC,aAAa,SAAS,MAAM,aAAa,EACvD,OAAO,EAAE,WAAW,CAAC,aAAa,CAAC,GACpC,OAAO,CAAC,QAAQ,CAAC;IAOpB,OAAO,CAAC,qBAAqB;IAqB7B;;OAEG;IACG,MAAM,CAAC,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,QAAQ,CAAC;CAOxD"}
|
package/index.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { Readable } from 'node:stream';
|
|
2
2
|
import { isGeneratorObject } from 'node:util/types';
|
|
3
|
+
import { ResponseWithUrl } from '@crawlee/core';
|
|
3
4
|
import { Impit } from 'impit';
|
|
4
5
|
import { LruCache } from '@apify/datastructures';
|
|
5
6
|
export const Browser = {
|
|
@@ -106,31 +107,9 @@ export class ImpitHttpClient {
|
|
|
106
107
|
* @inheritDoc
|
|
107
108
|
*/
|
|
108
109
|
async sendRequest(request) {
|
|
109
|
-
const { response
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
case 'text':
|
|
113
|
-
responseBody = await response.text();
|
|
114
|
-
break;
|
|
115
|
-
case 'json':
|
|
116
|
-
responseBody = await response.json();
|
|
117
|
-
break;
|
|
118
|
-
case 'buffer':
|
|
119
|
-
responseBody = await response.bytes();
|
|
120
|
-
break;
|
|
121
|
-
default:
|
|
122
|
-
throw new Error('Unsupported response type.');
|
|
123
|
-
}
|
|
124
|
-
return {
|
|
125
|
-
headers: Object.fromEntries(response.headers.entries()),
|
|
126
|
-
statusCode: response.status,
|
|
127
|
-
url: response.url,
|
|
128
|
-
request,
|
|
129
|
-
redirectUrls,
|
|
130
|
-
trailers: {},
|
|
131
|
-
body: responseBody,
|
|
132
|
-
complete: true,
|
|
133
|
-
};
|
|
110
|
+
const { response } = await this.getResponse(request);
|
|
111
|
+
// todo - cast shouldn't be needed here, impit returns `Uint8Array`
|
|
112
|
+
return new ResponseWithUrl((await response.bytes()), response);
|
|
134
113
|
}
|
|
135
114
|
getStreamWithProgress(response) {
|
|
136
115
|
const responseStream = Readable.fromWeb(response.body);
|
|
@@ -152,22 +131,10 @@ export class ImpitHttpClient {
|
|
|
152
131
|
* @inheritDoc
|
|
153
132
|
*/
|
|
154
133
|
async stream(request) {
|
|
155
|
-
const { response
|
|
156
|
-
const [stream
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
url: response.url,
|
|
160
|
-
statusCode: response.status,
|
|
161
|
-
stream,
|
|
162
|
-
complete: true,
|
|
163
|
-
get downloadProgress() {
|
|
164
|
-
return getDownloadProgress();
|
|
165
|
-
},
|
|
166
|
-
uploadProgress: { percent: 100, transferred: 0 },
|
|
167
|
-
redirectUrls,
|
|
168
|
-
headers: Object.fromEntries(response.headers.entries()),
|
|
169
|
-
trailers: {},
|
|
170
|
-
};
|
|
134
|
+
const { response } = await this.getResponse(request);
|
|
135
|
+
const [stream] = this.getStreamWithProgress(response);
|
|
136
|
+
// Cast shouldn't be needed here, undici might have a slightly different `ReadableStream` type
|
|
137
|
+
return new ResponseWithUrl(Readable.toWeb(stream), response);
|
|
171
138
|
}
|
|
172
139
|
}
|
|
173
140
|
//# sourceMappingURL=index.js.map
|
package/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAEvC,OAAO,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAEvC,OAAO,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AAEpD,OAAO,EAA6D,eAAe,EAAE,MAAM,eAAe,CAAC;AAE3G,OAAO,EAAE,KAAK,EAAE,MAAM,OAAO,CAAC;AAE9B,OAAO,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AAEjD,MAAM,CAAC,MAAM,OAAO,GAAG;IACnB,QAAQ,EAAE,QAAQ;IAClB,SAAS,EAAE,SAAS;CACd,CAAC;AAOX;;GAEG;AACH,MAAM,OAAO,eAAe;IAChB,YAAY,CAAe;IAC3B,YAAY,CAAS;IACrB,eAAe,CAAU;IAEjC;;;;;OAKG;IACK,WAAW,GAAoB,IAAI,QAAQ,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC,CAAC;IAE/D,SAAS,CAAC,OAAqB;QACnC,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QAEzC,IAAI,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;YACjC,OAAO,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAE,CAAC;QAC3C,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC;QAClC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QAEvC,OAAO,MAAM,CAAC;IAClB,CAAC;IAED,YAAY,OAAoE;QAC5E,IAAI,CAAC,YAAY,GAAG,OAAO,IAAI,EAAE,CAAC;QAElC,IAAI,CAAC,YAAY,GAAG,OAAO,EAAE,YAAY,IAAI,EAAE,CAAC;QAChD,IAAI,CAAC,eAAe,GAAG,OAAO,EAAE,eAAe,IAAI,IAAI,CAAC;IAC5D,CAAC;IAED;;;;OAIG;IACK,WAAW,CACf,OAAmE;QAEnE,IAAI,CAAC,OAAO,EAAE,CAAC;YACX,OAAO,SAAS,CAAC;QACrB,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;QAE7B,KAAK,MAAM,UAAU,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YAC5C,MAAM,WAAW,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;YAExC,KAAK,MAAM,KAAK,IAAI,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,EAAE,CAAC;gBAC3E,IAAI,KAAK,KAAK,SAAS;oBAAE,SAAS;gBAElC,MAAM,CAAC,MAAM,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;YACrC,CAAC;QACL,CAAC;QAED,OAAO,MAAM,CAAC;IAClB,CAAC;IAEO,aAAa,CACjB,IAA6D;QAE7D,IAAI,iBAAiB,CAAC,IAAI,CAAC,EAAE,CAAC;YAC1B,OAAO,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAQ,CAAC;QACtD,CAAC;QACD,IAAI,IAAI,YAAY,QAAQ,EAAE,CAAC;YAC3B,OAAO,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAQ,CAAC;QACvC,CAAC;QAED,OAAO,IAAW,CAAC;IACvB,CAAC;IAED;;;;OAIG;IACK,KAAK,CAAC,WAAW,CACrB,OAAmC,EACnC,SAGC;QAED,IAAI,CAAC,SAAS,EAAE,aAAa,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;YACtD,MAAM,IAAI,KAAK,CAAC,kCAAkC,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC;QAC5E,CAAC;QAED,MAAM,GAAG,GAAG,OAAO,OAAO,CAAC,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC;QAE7E,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC;YACzB,GAAG,IAAI,CAAC,YAAY;YACpB,QAAQ,EAAE,OAAO,CAAC,QAAQ;YAC1B,eAAe,EAAE,KAAK;SACzB,CAAC,CAAC;QAEH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE;YACpC,MAAM,EAAE,OAAO,CAAC,MAAoB;YACpC,OAAO,EAAE,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC;YAC1C,IAAI,EAAE,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,IAAI,CAAC;SACzC,CAAC,CAAC;QAEH,IAAI,IAAI,CAAC,eAAe,IAAI,QAAQ,CAAC,MAAM,IAAI,GAAG,IAAI,QAAQ,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;YAC1E,MAAM,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;YAClD,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,QAAQ,IAAI,EAAE,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC;YAEzD,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACZ,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC;YAClE,CAAC;YAED,OAAO,IAAI,CAAC,WAAW,CACnB;gBACI,GAAG,OAAO;gBACV,GAAG,EAAE,WAAW,CAAC,IAAI;aACxB,EACD;gBACI,aAAa,EAAE,CAAC,SAAS,EAAE,aAAa,IAAI,CAAC,CAAC,GAAG,CAAC;gBAClD,YAAY,EAAE,CAAC,GAAG,CAAC,SAAS,EAAE,YAAY,IAAI,EAAE,CAAC,EAAE,WAAW,CAAC;aAClE,CACJ,CAAC;QACN,CAAC;QAED,OAAO;YACH,QAAQ;YACR,YAAY,EAAE,SAAS,EAAE,YAAY,IAAI,EAAE;SAC9C,CAAC;IACN,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,WAAW,CACb,OAAmC;QAEnC,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;QAErD,mEAAmE;QACnE,OAAO,IAAI,eAAe,CAAC,CAAC,MAAM,QAAQ,CAAC,KAAK,EAAE,CAAQ,EAAE,QAAQ,CAAC,CAAC;IAC1E,CAAC;IAEO,qBAAqB,CACzB,QAAuB;QAEvB,MAAM,cAAc,GAAG,QAAQ,CAAC,OAAO,CAAC,QAAQ,CAAC,IAA2B,CAAC,CAAC;QAC9E,IAAI,WAAW,GAAG,CAAC,CAAC;QACpB,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC;QAClE,cAAc,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE;YAChC,WAAW,IAAI,KAAK,CAAC,MAAM,CAAC;QAChC,CAAC,CAAC,CAAC;QAEH,MAAM,mBAAmB,GAAG,GAAG,EAAE;YAC7B,OAAO;gBACH,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,WAAW,GAAG,KAAK,CAAC,GAAG,GAAG,CAAC;gBAChD,WAAW;gBACX,KAAK;aACR,CAAC;QACN,CAAC,CAAC;QAEF,OAAO,CAAC,cAAc,EAAE,mBAAmB,CAAC,CAAC;IACjD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM,CAAC,OAAoB;QAC7B,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;QACrD,MAAM,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,qBAAqB,CAAC,QAAQ,CAAC,CAAC;QAEtD,8FAA8F;QAC9F,OAAO,IAAI,eAAe,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAQ,EAAE,QAAQ,CAAC,CAAC;IACxE,CAAC;CACJ"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/impit-client",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.13",
|
|
4
4
|
"description": "impit-based HTTP client implementation for Crawlee. Impersonates browser requests to avoid bot detection.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.0.0"
|
|
@@ -63,5 +63,5 @@
|
|
|
63
63
|
}
|
|
64
64
|
}
|
|
65
65
|
},
|
|
66
|
-
"gitHead": "
|
|
66
|
+
"gitHead": "24df7c8dba6b6c9623c02ccc655d8ee748f003e3"
|
|
67
67
|
}
|