apify 4.0.0-beta.15 → 4.0.0-beta.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/apify_storage_client.js +43 -7
- package/package.json +1 -1
|
@@ -53,6 +53,27 @@ class PpeAwareDatasetClient extends ApifyDatasetClient {
|
|
|
53
53
|
context.chargeResult === undefined ? result : mergeChargeResults(context.chargeResult, result);
|
|
54
54
|
}
|
|
55
55
|
}
|
|
56
|
+
// crawlee v4's `StorageClient` sub-client interfaces use different method names
|
|
57
|
+
// than `apify-client`'s resource clients (`getValue`/`getRecord`,
|
|
58
|
+
// `pushData`/`pushItems`, `getData`/`listItems`, `getMetadata`/`get`,
|
|
59
|
+
// `drop`/`delete`). `adapt` wraps a client in a name-remapping proxy: `renames`
|
|
60
|
+
// aliases the differing methods and `overrides` replaces the few whose return
|
|
61
|
+
// shape differs; everything else — identically-named methods and the
|
|
62
|
+
// pay-per-event marker symbol — passes straight through.
|
|
63
|
+
//
|
|
64
|
+
// `purge()` has no apify-client equivalent and isn't needed on the platform
|
|
65
|
+
// (a run's storages are already fresh), so it's a no-op.
|
|
66
|
+
const noPurge = { purge: async () => { } };
|
|
67
|
+
function adapt(client, renames, overrides = {}) {
|
|
68
|
+
return new Proxy(client, {
|
|
69
|
+
get(target, prop) {
|
|
70
|
+
if (typeof prop === 'string' && prop in overrides)
|
|
71
|
+
return overrides[prop];
|
|
72
|
+
const value = Reflect.get(target, (typeof prop === 'string' && renames[prop]) || prop, target);
|
|
73
|
+
return typeof value === 'function' ? value.bind(target) : value;
|
|
74
|
+
},
|
|
75
|
+
});
|
|
76
|
+
}
|
|
56
77
|
/**
|
|
57
78
|
* Bridges `apify-client`'s synchronous resource accessors (`dataset(id)`,
|
|
58
79
|
* `keyValueStore(id)`, `requestQueue(id, options?)`) to crawlee v4's
|
|
@@ -85,19 +106,34 @@ export class ApifyStorageClient {
|
|
|
85
106
|
}
|
|
86
107
|
async createDatasetClient(options) {
|
|
87
108
|
const id = await this.resolveId(options, 'Dataset');
|
|
88
|
-
const
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
109
|
+
const client = this.chargingDatasetClient(id) ?? this.client.dataset(id);
|
|
110
|
+
return adapt(client, {
|
|
111
|
+
getMetadata: 'get',
|
|
112
|
+
drop: 'delete',
|
|
113
|
+
pushData: 'pushItems',
|
|
114
|
+
getData: 'listItems',
|
|
115
|
+
}, noPurge);
|
|
93
116
|
}
|
|
94
117
|
async createKeyValueStoreClient(options) {
|
|
95
118
|
const id = await this.resolveId(options, 'KeyValueStore');
|
|
96
|
-
|
|
119
|
+
const client = this.client.keyValueStore(id);
|
|
120
|
+
return adapt(client, {
|
|
121
|
+
getMetadata: 'get',
|
|
122
|
+
getValue: 'getRecord',
|
|
123
|
+
setValue: 'setRecord',
|
|
124
|
+
deleteValue: 'deleteRecord',
|
|
125
|
+
drop: 'delete',
|
|
126
|
+
getPublicUrl: 'getRecordPublicUrl',
|
|
127
|
+
}, {
|
|
128
|
+
...noPurge,
|
|
129
|
+
// crawlee expects an array; apify-client returns `{ items }`.
|
|
130
|
+
listKeys: async (opts) => (await client.listKeys(opts)).items,
|
|
131
|
+
});
|
|
97
132
|
}
|
|
98
133
|
async createRequestQueueClient(options) {
|
|
99
134
|
const id = await this.resolveId(options, 'RequestQueue');
|
|
100
|
-
|
|
135
|
+
const client = this.client.requestQueue(id, options?.clientKey ? { clientKey: options.clientKey } : undefined);
|
|
136
|
+
return adapt(client, { getMetadata: 'get', drop: 'delete' }, noPurge);
|
|
101
137
|
}
|
|
102
138
|
/**
|
|
103
139
|
* Returns a charging-aware dataset client when `id` is the run's default
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "apify",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.16",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.0.0"
|