litescrape-sdk 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +283 -0
- package/dist/cjs/batch.d.ts +12 -0
- package/dist/cjs/batch.d.ts.map +1 -0
- package/dist/cjs/batch.js +131 -0
- package/dist/cjs/batch.js.map +1 -0
- package/dist/cjs/cache.d.ts +21 -0
- package/dist/cjs/cache.d.ts.map +1 -0
- package/dist/cjs/cache.js +112 -0
- package/dist/cjs/cache.js.map +1 -0
- package/dist/cjs/client.d.ts +38 -0
- package/dist/cjs/client.d.ts.map +1 -0
- package/dist/cjs/client.js +143 -0
- package/dist/cjs/client.js.map +1 -0
- package/dist/cjs/errors.d.ts +53 -0
- package/dist/cjs/errors.d.ts.map +1 -0
- package/dist/cjs/errors.js +121 -0
- package/dist/cjs/errors.js.map +1 -0
- package/dist/cjs/index.d.ts +8 -0
- package/dist/cjs/index.d.ts.map +1 -0
- package/dist/cjs/index.js +40 -0
- package/dist/cjs/index.js.map +1 -0
- package/dist/cjs/models.d.ts +1735 -0
- package/dist/cjs/models.d.ts.map +1 -0
- package/dist/cjs/models.js +389 -0
- package/dist/cjs/models.js.map +1 -0
- package/dist/cjs/package.json +1 -0
- package/dist/cjs/result.d.ts +23 -0
- package/dist/cjs/result.d.ts.map +1 -0
- package/dist/cjs/result.js +49 -0
- package/dist/cjs/result.js.map +1 -0
- package/dist/cjs/runtime.d.ts +53 -0
- package/dist/cjs/runtime.d.ts.map +1 -0
- package/dist/cjs/runtime.js +237 -0
- package/dist/cjs/runtime.js.map +1 -0
- package/dist/cjs/stores.d.ts +143 -0
- package/dist/cjs/stores.d.ts.map +1 -0
- package/dist/cjs/stores.js +194 -0
- package/dist/cjs/stores.js.map +1 -0
- package/dist/cjs/validation.d.ts +65 -0
- package/dist/cjs/validation.d.ts.map +1 -0
- package/dist/cjs/validation.js +122 -0
- package/dist/cjs/validation.js.map +1 -0
- package/dist/cjs/version.d.ts +2 -0
- package/dist/cjs/version.d.ts.map +1 -0
- package/dist/cjs/version.js +5 -0
- package/dist/cjs/version.js.map +1 -0
- package/dist/esm/batch.d.ts +12 -0
- package/dist/esm/batch.d.ts.map +1 -0
- package/dist/esm/batch.js +128 -0
- package/dist/esm/batch.js.map +1 -0
- package/dist/esm/cache.d.ts +21 -0
- package/dist/esm/cache.d.ts.map +1 -0
- package/dist/esm/cache.js +106 -0
- package/dist/esm/cache.js.map +1 -0
- package/dist/esm/client.d.ts +38 -0
- package/dist/esm/client.d.ts.map +1 -0
- package/dist/esm/client.js +138 -0
- package/dist/esm/client.js.map +1 -0
- package/dist/esm/errors.d.ts +53 -0
- package/dist/esm/errors.d.ts.map +1 -0
- package/dist/esm/errors.js +106 -0
- package/dist/esm/errors.js.map +1 -0
- package/dist/esm/index.d.ts +8 -0
- package/dist/esm/index.d.ts.map +1 -0
- package/dist/esm/index.js +6 -0
- package/dist/esm/index.js.map +1 -0
- package/dist/esm/models.d.ts +1735 -0
- package/dist/esm/models.d.ts.map +1 -0
- package/dist/esm/models.js +384 -0
- package/dist/esm/models.js.map +1 -0
- package/dist/esm/result.d.ts +23 -0
- package/dist/esm/result.d.ts.map +1 -0
- package/dist/esm/result.js +45 -0
- package/dist/esm/result.js.map +1 -0
- package/dist/esm/runtime.d.ts +53 -0
- package/dist/esm/runtime.d.ts.map +1 -0
- package/dist/esm/runtime.js +226 -0
- package/dist/esm/runtime.js.map +1 -0
- package/dist/esm/stores.d.ts +143 -0
- package/dist/esm/stores.d.ts.map +1 -0
- package/dist/esm/stores.js +190 -0
- package/dist/esm/stores.js.map +1 -0
- package/dist/esm/validation.d.ts +65 -0
- package/dist/esm/validation.d.ts.map +1 -0
- package/dist/esm/validation.js +109 -0
- package/dist/esm/validation.js.map +1 -0
- package/dist/esm/version.d.ts +2 -0
- package/dist/esm/version.d.ts.map +1 -0
- package/dist/esm/version.js +2 -0
- package/dist/esm/version.js.map +1 -0
- package/package.json +74 -0
- package/src/batch.ts +158 -0
- package/src/cache.ts +123 -0
- package/src/client.ts +199 -0
- package/src/errors.ts +120 -0
- package/src/index.ts +17 -0
- package/src/models.ts +570 -0
- package/src/package.json +3 -0
- package/src/result.ts +49 -0
- package/src/runtime.ts +255 -0
- package/src/stores.ts +206 -0
- package/src/validation.ts +140 -0
- package/src/version.ts +1 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Litescrape
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,283 @@
|
|
|
1
|
+
# litescrape-sdk
|
|
2
|
+
|
|
3
|
+
TypeScript and JavaScript SDK for the [Litescrape API](https://litescrape.com): validated,
|
|
4
|
+
concurrent, retrying calls to all 34 search, maps, reviews, Google Play, and Apple App Store
|
|
5
|
+
endpoints, with results returned in input order.
|
|
6
|
+
|
|
7
|
+
Follows the contracts and behavior of the [Python SDK](https://github.com/litescrape/litescrape-sdk)
|
|
8
|
+
v0.5.2. Supports Node.js 22.13+ with ESM, CommonJS, and TypeScript declarations. This is a
|
|
9
|
+
server-side SDK: keep your API key on the server.
|
|
10
|
+
|
|
11
|
+
Until the first npm release, install from GitHub with an account that has repository access
|
|
12
|
+
(npm builds the package during installation):
|
|
13
|
+
|
|
14
|
+
```sh
|
|
15
|
+
npm install git+https://github.com/litescrape/litescrape-sdk-ts.git
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
After publication, the npm package name is `litescrape-sdk`:
|
|
19
|
+
|
|
20
|
+
```sh
|
|
21
|
+
npm install litescrape-sdk
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
```typescript
|
|
25
|
+
import { GoogleMaps, scrape } from "litescrape-sdk";
|
|
26
|
+
|
|
27
|
+
// Set LITESCRAPE_API_KEY in your environment, or pass { apiKey: "ls_live_..." }.
|
|
28
|
+
const results = await scrape([
|
|
29
|
+
{ endpoint: "google_search", q: "coffee grinders", gl: "us" },
|
|
30
|
+
new GoogleMaps({ q: "coffee", type: "search", ll: "@40.745,-73.988,14z" }),
|
|
31
|
+
]);
|
|
32
|
+
|
|
33
|
+
for (const result of results) {
|
|
34
|
+
console.log(result.ok, result.data ?? result.error);
|
|
35
|
+
}
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
CommonJS: `const { GoogleMaps, scrape } = require("litescrape-sdk");`.
|
|
39
|
+
All network calls return promises. `ascrape` is an alias of `scrape`.
|
|
40
|
+
|
|
41
|
+
## Requests and results
|
|
42
|
+
|
|
43
|
+
Pass plain objects with an `endpoint` discriminator or construct typed request objects such as
|
|
44
|
+
`new GoogleSearch({ q: "coffee" })`. Constructors and `scrape` validate at runtime; unknown
|
|
45
|
+
parameters, invalid values, missing required fields, and incompatible parameter combinations
|
|
46
|
+
are rejected. Every input is validated before the first HTTP call. A `ValidationError` contains
|
|
47
|
+
`problems`, an array of `[inputIndex, message]` pairs. `parseRequest(unknown)` validates external input.
|
|
48
|
+
|
|
49
|
+
`REQUEST_TYPES` maps every endpoint slug to its request constructor. Constructors expose `path`
|
|
50
|
+
and `parameters`; instances expose immutable normalized fields, `path`, and `queryParams()`.
|
|
51
|
+
Use strings or `bigint` for numeric IDs larger than JavaScript's safe integer range. Store IDs
|
|
52
|
+
are strictly strings, matching the Python Store contracts.
|
|
53
|
+
|
|
54
|
+
Each `Result` has `index`, `request`, `ok`, `data`, `error`, `statusCode`, `requestId`, `attempts`,
|
|
55
|
+
`elapsed` (seconds), and `jobId` (empty outside batch mode). `raiseForError()` throws the error or
|
|
56
|
+
returns the data. Response data is a `Record<string, unknown>`; provider response fields are not
|
|
57
|
+
validated or discarded.
|
|
58
|
+
|
|
59
|
+
The Python names `query_params()`, `raise_for_error()`, `status_code`, `request_id`, `job_id`,
|
|
60
|
+
`error_code`, and `retry_after` are also available. SDK options use camelCase; provider parameters
|
|
61
|
+
retain their API spelling, such as `place_id`, `fast_mode`, and `safeSearch`.
|
|
62
|
+
|
|
63
|
+
```typescript
|
|
64
|
+
import { keyStatus } from "litescrape-sdk";
|
|
65
|
+
|
|
66
|
+
const status = await keyStatus(); // Unbilled GET /api/keys/status
|
|
67
|
+
console.log(status.remaining_calls, status.concurrency_limit);
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
`key_status` and `akey_status` are aliases of `keyStatus` and take the same options object.
|
|
71
|
+
|
|
72
|
+
## Options
|
|
73
|
+
|
|
74
|
+
```typescript
|
|
75
|
+
await scrape(requests, {
|
|
76
|
+
apiKey: process.env.LITESCRAPE_API_KEY,
|
|
77
|
+
baseUrl: "https://api.litescrape.com",
|
|
78
|
+
attempts: 5,
|
|
79
|
+
concurrency: 10,
|
|
80
|
+
timeout: 120,
|
|
81
|
+
requestTimeout: 15,
|
|
82
|
+
signal: AbortSignal.timeout(60_000),
|
|
83
|
+
onProgress: (completed, total, result) => console.log(completed, total, result.ok),
|
|
84
|
+
});
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
| Option | Default | Meaning |
|
|
88
|
+
| ---------------- | ---------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------ |
|
|
89
|
+
| `apiKey` | `LITESCRAPE_API_KEY` | Bearer key. |
|
|
90
|
+
| `baseUrl` | `LITESCRAPE_API_URL` or `https://api.litescrape.com` | API URL. |
|
|
91
|
+
| `attempts` | `5` | Total attempts per HTTP operation; `1` disables retries. |
|
|
92
|
+
| `concurrency` | Key limit, or `25`; batch mode `32` | Synchronous calls can lower, never raise, the key limit. Batch submission/poll concurrency is independent of server execution. |
|
|
93
|
+
| `timeout` | `120` | HTTP transport timeout in seconds for each attempt, including reading its body. |
|
|
94
|
+
| `requestTimeout` | Omitted | Default server deadline in seconds: greater than zero, at most 90. |
|
|
95
|
+
| `batched` | `false` | Submit durable server jobs, then poll. |
|
|
96
|
+
| `useCache` | `false` | Recover saved durable jobs; requires `batched: true`. |
|
|
97
|
+
| `cachePath` | See below | SQLite job cache. |
|
|
98
|
+
| `signal` | Omitted | Abort local requests, queued work, retries, and polling. |
|
|
99
|
+
| `onProgress` | Omitted | Called once per result in completion order. Returned results remain in input order. |
|
|
100
|
+
| `fetch` | Native `fetch` | Custom fetch-compatible transport, including for tests. |
|
|
101
|
+
|
|
102
|
+
There is no automatic terminal progress bar. `keyStatus` accepts `apiKey`, `baseUrl`, `attempts`,
|
|
103
|
+
`timeout`, `signal`, and `fetch`.
|
|
104
|
+
|
|
105
|
+
Before synchronous scrapes, the SDK checks the key's balance and concurrency limit. Too few
|
|
106
|
+
remaining calls raises `PaymentRequiredError` before scraping. Overlapping `scrape` calls in the
|
|
107
|
+
same Node.js runtime share the key limit. Worker threads and separate processes have independent
|
|
108
|
+
limits. The server remains the final authority on concurrency.
|
|
109
|
+
|
|
110
|
+
Retries cover connection failures, transport timeouts, unusable success bodies, HTTP 429/5xx,
|
|
111
|
+
retryable envelopes, and the Python SDK's transient-capacity error codes. Other 400/401/402/403/404/422
|
|
112
|
+
responses are not retried unless the API explicitly marks them retryable. Backoff grows exponentially
|
|
113
|
+
with jitter and honors `Retry-After`, capped at 30 seconds. A 401, 402, or `api_key_disabled` error stops
|
|
114
|
+
unsent synchronous work. Each affected item still gets a `Result`.
|
|
115
|
+
|
|
116
|
+
Per-item failures appear as `Result.error`. Setup/status failures reject the entire call. Exported
|
|
117
|
+
errors are `LitescrapeError`, `ValidationError`, `TransportError`, `APIError`, `AuthenticationError`,
|
|
118
|
+
`PaymentRequiredError`, `NotFoundError`, `RateLimitError`, and `RequestDeadlineExceededError`.
|
|
119
|
+
API errors preserve `statusCode`, `errorCode`, `requestId`, `retryable`, `retryAfter`, and `body`.
|
|
120
|
+
Cancellation rejects with the abort signal's reason. A thrown `onProgress` callback rejects the call;
|
|
121
|
+
avoid throwing from progress callbacks.
|
|
122
|
+
|
|
123
|
+
Only successful API attempts are billed. A synchronous response lost in transit may already have
|
|
124
|
+
been billed, and retrying can incur another charge. Use durable batches for idempotent submission.
|
|
125
|
+
|
|
126
|
+
## Durable batches and recovery
|
|
127
|
+
|
|
128
|
+
Requires an API deployment with durable batch support. The SDK saves the entire workload's
|
|
129
|
+
submission intentions and batch IDs in a SQLite transaction before sending the first input, saves
|
|
130
|
+
acknowledged job IDs, submits all inputs, and then polls. Accepted jobs keep running if your process
|
|
131
|
+
exits or your connection drops. Results are retained for 24 hours after completion.
|
|
132
|
+
|
|
133
|
+
```typescript
|
|
134
|
+
import { GoogleSearch, scrape } from "litescrape-sdk";
|
|
135
|
+
|
|
136
|
+
const queries = [new GoogleSearch({ q: "coffee grinders" }), new GoogleSearch({ q: "espresso machines" })];
|
|
137
|
+
const options = { batched: true, cachePath: "./my-batch.sqlite3" };
|
|
138
|
+
const results = await scrape(queries, options);
|
|
139
|
+
|
|
140
|
+
// After a crash or disconnect, rerun with the same inputs, key, and cache path:
|
|
141
|
+
const recovered = await scrape(queries, { ...options, useCache: true });
|
|
142
|
+
for (const result of recovered) console.log(result.jobId, result.data ?? result.error);
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
`useCache: false` creates fresh jobs. `useCache: true` matches by API key, API URL, endpoint, normalized
|
|
146
|
+
parameters, and the occurrence number of duplicate inputs. Distinct inputs can be reordered.
|
|
147
|
+
Submission retries and recovery after a lost acknowledgement reuse their idempotency key and batch
|
|
148
|
+
ID. Replacements are created only after the server confirms a job is missing or expired; a polling
|
|
149
|
+
outage does not authorize another charge. Keep separate cache files for independently resumable batches.
|
|
150
|
+
|
|
151
|
+
The default cache is `~/.cache/litescrape/jobs-ts.sqlite3`, overridden by `LITESCRAPE_JOB_CACHE` or
|
|
152
|
+
`cachePath`. It stores request fingerprints and job/submission IDs, without raw API keys, query text,
|
|
153
|
+
or responses. The TypeScript and Python cache files are separate and are not interchangeable.
|
|
154
|
+
Batch storage uses Node's built-in [SQLite module](https://nodejs.org/api/sqlite.html), which may emit
|
|
155
|
+
an experimental-feature warning on Node 22/24. No native npm add-on is needed.
|
|
156
|
+
|
|
157
|
+
New jobs reserve one credit; terminal failures refund that reservation. Polling and retrying the same
|
|
158
|
+
submission do not consume more credits. Batch mode skips the synchronous balance check, so previously
|
|
159
|
+
paid jobs can be recovered with zero credits. Check each result for submission errors, including
|
|
160
|
+
insufficient credits. `attempts` controls submission/poll transport retries; provider retries and
|
|
161
|
+
execution capacity are managed by the server. `Result.attempts` reports the final HTTP operation's
|
|
162
|
+
attempt count, matching Python; it is not a sum of polls or server worker attempts.
|
|
163
|
+
|
|
164
|
+
## Google Search fast mode and request deadlines
|
|
165
|
+
|
|
166
|
+
`new GoogleSearch({ q: "coffee", fast_mode: true })` requests organic results plus metadata and
|
|
167
|
+
parameters. It skips AI Overview and other response groups. The default returns the full response;
|
|
168
|
+
fast mode is unavailable on `GoogleAiOverview`.
|
|
169
|
+
|
|
170
|
+
```typescript
|
|
171
|
+
import { GoogleSearch, RequestDeadlineExceededError, scrape } from "litescrape-sdk";
|
|
172
|
+
|
|
173
|
+
const [result] = await scrape([new GoogleSearch({ q: "coffee grinders", timeout: 10 })], {
|
|
174
|
+
requestTimeout: 15,
|
|
175
|
+
attempts: 1,
|
|
176
|
+
});
|
|
177
|
+
if (result.error instanceof RequestDeadlineExceededError) {
|
|
178
|
+
console.log(result.error.requestId, result.error.retryable, result.error.message);
|
|
179
|
+
} else {
|
|
180
|
+
console.log(result.raiseForError());
|
|
181
|
+
}
|
|
182
|
+
```
|
|
183
|
+
|
|
184
|
+
An item's `timeout` overrides `requestTimeout`; both accept fractional seconds greater than zero
|
|
185
|
+
and at most 90. The server deadline covers each synchronous attempt from gateway receipt through
|
|
186
|
+
admission, scraping, and billing. Expiry returns HTTP 503 `request_deadline_exceeded`, is retryable,
|
|
187
|
+
and that attempt is not charged. In batch mode it limits each worker attempt after queueing.
|
|
188
|
+
|
|
189
|
+
SDK queueing, the status check, transport, backoff, and the full workload are outside that server
|
|
190
|
+
deadline. The SDK option `timeout` remains the HTTP transport timeout; keep it longer than the server
|
|
191
|
+
deadline. A `TransportError` alone does not establish whether an attempt was billed.
|
|
192
|
+
|
|
193
|
+
## Endpoints
|
|
194
|
+
|
|
195
|
+
| Request class | Endpoint slug |
|
|
196
|
+
| --------------------------- | ------------------------------- |
|
|
197
|
+
| `GoogleSearch` | `google_search` |
|
|
198
|
+
| `GoogleAiOverview` | `google_ai_overview` |
|
|
199
|
+
| `GoogleAiMode` | `google_ai_mode` |
|
|
200
|
+
| `GoogleAds` | `google_ads` |
|
|
201
|
+
| `GoogleShopping` | `google_shopping` |
|
|
202
|
+
| `GoogleShoppingProduct` | `google_shopping_product` |
|
|
203
|
+
| `GoogleLocal` | `google_local` |
|
|
204
|
+
| `GoogleMaps` | `google_maps` |
|
|
205
|
+
| `GoogleMapsLiveFootTraffic` | `google_maps_live_foot_traffic` |
|
|
206
|
+
| `GoogleMapsPosts` | `google_maps_posts` |
|
|
207
|
+
| `GoogleMapsPhoto` | `google_maps_photo` |
|
|
208
|
+
| `GoogleMapsWebResults` | `google_maps_web_results` |
|
|
209
|
+
| `GoogleReviews` | `google_reviews` |
|
|
210
|
+
| `GoogleContributorReviews` | `google_contributor_reviews` |
|
|
211
|
+
| `BingSearch` | `bing_search` |
|
|
212
|
+
| `BingMaps` | `bing_maps` |
|
|
213
|
+
| `DuckDuckGoSearch` | `duckduckgo_search` |
|
|
214
|
+
| `DuckDuckGoMaps` | `duckduckgo_maps` |
|
|
215
|
+
| `YelpSearch` | `yelp_search` |
|
|
216
|
+
| `YelpReviews` | `yelp_reviews` |
|
|
217
|
+
| `TripadvisorSearch` | `tripadvisor_search` |
|
|
218
|
+
| `TripadvisorPlace` | `tripadvisor_place` |
|
|
219
|
+
| `TripadvisorReviews` | `tripadvisor_reviews` |
|
|
220
|
+
| `AppleMapsPlaces` | `apple_maps_places` |
|
|
221
|
+
| `AppleMapsReviews` | `apple_maps_reviews` |
|
|
222
|
+
| `GooglePlayApps` | `google_play_apps` |
|
|
223
|
+
| `GooglePlayGames` | `google_play_games` |
|
|
224
|
+
| `GooglePlayBooks` | `google_play_books` |
|
|
225
|
+
| `GooglePlayMovies` | `google_play_movies` |
|
|
226
|
+
| `GooglePlayProduct` | `google_play_product` |
|
|
227
|
+
| `GooglePlayReviews` | `google_play_reviews` |
|
|
228
|
+
| `AppleAppStoreSearch` | `apple_app_store_search` |
|
|
229
|
+
| `AppleAppStoreProduct` | `apple_app_store_product` |
|
|
230
|
+
| `AppleAppStoreReviews` | `apple_app_store_reviews` |
|
|
231
|
+
|
|
232
|
+
## Store APIs (Alpha)
|
|
233
|
+
|
|
234
|
+
All nine Store operations use your existing key. Fields depend on the storefront's response.
|
|
235
|
+
|
|
236
|
+
```typescript
|
|
237
|
+
import { GooglePlayApps, GooglePlayProduct, AppleAppStoreSearch, scrape } from "litescrape-sdk";
|
|
238
|
+
|
|
239
|
+
const results = await scrape([
|
|
240
|
+
new GooglePlayApps({ q: "coffee", hl: "en", gl: "us" }),
|
|
241
|
+
new GooglePlayProduct({ product_id: "com.duolingo" }),
|
|
242
|
+
new AppleAppStoreSearch({ term: "coffee", country: "us", num: 10 }),
|
|
243
|
+
]);
|
|
244
|
+
```
|
|
245
|
+
|
|
246
|
+
Follow `litescrape_pagination.next` or use returned continuation tokens with the same operation and
|
|
247
|
+
parameters. Google Play's `chart`, `next_page_token`, `section_page_token`, and `see_more_token` are
|
|
248
|
+
mutually exclusive. Queries exclude category filters and charts. Apps/games charts require an omitted
|
|
249
|
+
device or `store_device: "phone"`; omit `store_device` with a query or category. Games queries use shared
|
|
250
|
+
Android app search; omit `q` or choose `games_category` to browse games.
|
|
251
|
+
|
|
252
|
+
Search text is limited to 2,048 UTF-8 bytes; Apple terms also have a 4,096 URL-encoded-byte limit.
|
|
253
|
+
Apple review pages start at one; exhausted pages return an empty list. Successful pages consume one
|
|
254
|
+
call. `search_metadata.raw_file` and `prettify_file`, when present, link to authenticated response
|
|
255
|
+
artifacts. Download them with the same bearer key; downloads are unbilled.
|
|
256
|
+
|
|
257
|
+
See the [API reference](https://litescrape.com/docs) for provider parameters and response groups.
|
|
258
|
+
|
|
259
|
+
## Development and releases
|
|
260
|
+
|
|
261
|
+
```sh
|
|
262
|
+
npm ci
|
|
263
|
+
npm run check
|
|
264
|
+
npm pack
|
|
265
|
+
npm run test:package
|
|
266
|
+
```
|
|
267
|
+
|
|
268
|
+
`npm run format` formats the source. Tests use local mock transports and a loopback HTTP server;
|
|
269
|
+
they require no API key or paid requests. The tests include all 34 endpoint allowlists, the Python
|
|
270
|
+
SDK's 547 Store boundary cases, recovery/idempotency scenarios, cancellation, ESM/CommonJS consumers,
|
|
271
|
+
and positive/negative TypeScript type checks. CI runs on Linux and Windows, including Node 22.13,
|
|
272
|
+
22, 24, and 26. The package smoke test installs the tarball into a fresh consumer without build scripts.
|
|
273
|
+
|
|
274
|
+
Update both `package.json` and `src/version.ts` for a release; regenerate the lockfile. A `v*` tag
|
|
275
|
+
triggers tests, package verification, and the publish workflow, which requires the tag to match the
|
|
276
|
+
package version. No tag or npm release is created by normal pushes.
|
|
277
|
+
|
|
278
|
+
Before the first automated release, publish the initial package with an authorized npm account,
|
|
279
|
+
then configure [npm trusted publishing](https://docs.npmjs.com/trusted-publishers/) for GitHub repository
|
|
280
|
+
`litescrape/litescrape-sdk-ts`, workflow `publish.yml`, and environment `npm`. Configure that GitHub
|
|
281
|
+
environment as desired. Subsequent tagged releases use OIDC; no npm token is stored in this
|
|
282
|
+
repository. The workflow enables provenance when the source repository is public and disables it
|
|
283
|
+
while the repository is private, as required by npm.
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { AnyRequest } from "./models.js";
|
|
2
|
+
import { Result } from "./result.js";
|
|
3
|
+
import { type RuntimeOptions } from "./runtime.js";
|
|
4
|
+
interface BatchOptions extends RuntimeOptions {
|
|
5
|
+
concurrency?: number;
|
|
6
|
+
useCache: boolean;
|
|
7
|
+
cachePath?: string;
|
|
8
|
+
onResult: (result: Result) => void;
|
|
9
|
+
}
|
|
10
|
+
export declare function runBatch(items: readonly AnyRequest[], options: BatchOptions): Promise<Result[]>;
|
|
11
|
+
export {};
|
|
12
|
+
//# sourceMappingURL=batch.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"batch.d.ts","sourceRoot":"","sources":["../../src/batch.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAC9C,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACrC,OAAO,EAOL,KAAK,cAAc,EACpB,MAAM,cAAc,CAAC;AAEtB,UAAU,YAAa,SAAQ,cAAc;IAC3C,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,EAAE,OAAO,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,IAAI,CAAC;CACpC;AAGD,wBAAsB,QAAQ,CAAC,KAAK,EAAE,SAAS,UAAU,EAAE,EAAE,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAuIrG"}
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.runBatch = runBatch;
|
|
4
|
+
const errors_js_1 = require("./errors.js");
|
|
5
|
+
const cache_js_1 = require("./cache.js");
|
|
6
|
+
const result_js_1 = require("./result.js");
|
|
7
|
+
const runtime_js_1 = require("./runtime.js");
|
|
8
|
+
async function runBatch(items, options) {
|
|
9
|
+
options.signal?.throwIfAborted();
|
|
10
|
+
const cache = new cache_js_1.JobCache(options.cachePath);
|
|
11
|
+
const keys = (0, cache_js_1.cacheKeys)(items, options.apiKey, options.baseUrl);
|
|
12
|
+
const limit = options.concurrency ?? 32;
|
|
13
|
+
const semaphores = [new runtime_js_1.Semaphore(limit)];
|
|
14
|
+
const started = performance.now();
|
|
15
|
+
const batchKey = (0, cache_js_1.newId)();
|
|
16
|
+
const indices = items.map((_, i) => i);
|
|
17
|
+
const results = new Array(items.length);
|
|
18
|
+
try {
|
|
19
|
+
const entries = cache.prepareMany(keys, options.useCache, batchKey, items.length);
|
|
20
|
+
const pollOnce = (jobId) => (0, runtime_js_1.requestWithRetries)(`/api/batch/jobs/${encodeURIComponent(jobId)}`, { ...options, semaphores });
|
|
21
|
+
const replace = (index) => cache.prepareMany([keys[index]], false, batchKey, items.length)[0];
|
|
22
|
+
const submit = async (index, replaceExpired = false) => {
|
|
23
|
+
let entry = replaceExpired ? replace(index) : entries[index];
|
|
24
|
+
if (entry.job_id) {
|
|
25
|
+
const outcome = await pollOnce(entry.job_id);
|
|
26
|
+
// Only positive evidence of expiry/missing jobs permits a fresh reservation.
|
|
27
|
+
if ((!outcome.error && outcome.data?.state !== "expired") ||
|
|
28
|
+
(outcome.error && ![404, 410].includes(outcome.statusCode ?? 0)))
|
|
29
|
+
return { entry, outcome };
|
|
30
|
+
entry = replace(index);
|
|
31
|
+
}
|
|
32
|
+
const outcome = await (0, runtime_js_1.requestWithRetries)(items[index].path + "/async", {
|
|
33
|
+
...options,
|
|
34
|
+
semaphores,
|
|
35
|
+
method: "POST",
|
|
36
|
+
body: items[index].queryParams(),
|
|
37
|
+
acceptedStatuses: [200, 202],
|
|
38
|
+
headers: {
|
|
39
|
+
"Idempotency-Key": entry.submission,
|
|
40
|
+
"X-Litescrape-Batch-ID": entry.batch_key,
|
|
41
|
+
"X-Litescrape-Batch-Size": String(entry.batch_size),
|
|
42
|
+
},
|
|
43
|
+
});
|
|
44
|
+
if (!outcome.error) {
|
|
45
|
+
const job = outcome.data;
|
|
46
|
+
if (typeof job.job_id !== "string" || !job.job_id)
|
|
47
|
+
outcome.error = new errors_js_1.TransportError("Batch submission returned no job ID.", { retryable: true });
|
|
48
|
+
else {
|
|
49
|
+
entry.job_id = job.job_id;
|
|
50
|
+
cache.save(keys[index], entry.submission, job.job_id, job.expires_at);
|
|
51
|
+
if (job.state === "expired" && options.useCache && !replaceExpired)
|
|
52
|
+
return submit(index, true);
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
return { entry, outcome };
|
|
56
|
+
};
|
|
57
|
+
const submissions = new Array(items.length);
|
|
58
|
+
await (0, runtime_js_1.each)(indices, limit, async (index) => {
|
|
59
|
+
submissions[index] = await submit(index);
|
|
60
|
+
});
|
|
61
|
+
const resolve = (index, { entry, outcome }) => {
|
|
62
|
+
if (!outcome.error) {
|
|
63
|
+
const job = outcome.data;
|
|
64
|
+
if (typeof job.job_id !== "string" || !job.job_id || job.job_id !== entry.job_id) {
|
|
65
|
+
outcome.error = new errors_js_1.TransportError("Batch job returned an invalid job ID.", { retryable: false });
|
|
66
|
+
}
|
|
67
|
+
else if (job.state === "succeeded" || job.state === "failed") {
|
|
68
|
+
const status = job.status_code;
|
|
69
|
+
const payload = job.response;
|
|
70
|
+
if (typeof status !== "number" ||
|
|
71
|
+
!Number.isInteger(status) ||
|
|
72
|
+
status < 100 ||
|
|
73
|
+
status > 599 ||
|
|
74
|
+
!(0, errors_js_1.isObject)(payload) ||
|
|
75
|
+
(job.state === "succeeded") !== (status === 200)) {
|
|
76
|
+
outcome.error = new errors_js_1.TransportError("Batch job returned an invalid response.", {
|
|
77
|
+
retryable: false,
|
|
78
|
+
});
|
|
79
|
+
}
|
|
80
|
+
else {
|
|
81
|
+
cache.save(keys[index], entry.submission, job.job_id, job.expires_at);
|
|
82
|
+
return new result_js_1.Result(index, items[index], {
|
|
83
|
+
data: status === 200 ? payload : null,
|
|
84
|
+
error: status === 200 ? null : (0, errors_js_1.apiError)(status, payload),
|
|
85
|
+
statusCode: status,
|
|
86
|
+
requestId: job.job_id,
|
|
87
|
+
attempts: outcome.attempts,
|
|
88
|
+
}, (performance.now() - started) / 1000, job.job_id);
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
else if (job.state === "expired") {
|
|
92
|
+
outcome.error = new errors_js_1.APIError("The retained batch response expired.", {
|
|
93
|
+
statusCode: 410,
|
|
94
|
+
errorCode: "job_expired",
|
|
95
|
+
requestId: entry.job_id ?? "",
|
|
96
|
+
});
|
|
97
|
+
outcome.statusCode = 410;
|
|
98
|
+
}
|
|
99
|
+
else if (job.state === "queued" || job.state === "running")
|
|
100
|
+
return null;
|
|
101
|
+
else
|
|
102
|
+
outcome.error = new errors_js_1.TransportError("Batch job returned an unknown state.", { retryable: false });
|
|
103
|
+
}
|
|
104
|
+
return new result_js_1.Result(index, items[index], { ...outcome, data: null, requestId: entry.job_id ?? outcome.requestId }, (performance.now() - started) / 1000, entry.job_id ?? "");
|
|
105
|
+
};
|
|
106
|
+
let pending = indices;
|
|
107
|
+
let polls = 0;
|
|
108
|
+
while (pending.length) {
|
|
109
|
+
await (0, runtime_js_1.each)(pending, limit, async (index) => {
|
|
110
|
+
let result = resolve(index, submissions[index]);
|
|
111
|
+
if (!result) {
|
|
112
|
+
const entry = submissions[index].entry;
|
|
113
|
+
submissions[index] = { entry, outcome: await pollOnce(entry.job_id) };
|
|
114
|
+
result = resolve(index, submissions[index]);
|
|
115
|
+
}
|
|
116
|
+
if (result) {
|
|
117
|
+
results[index] = result;
|
|
118
|
+
options.onResult(result);
|
|
119
|
+
}
|
|
120
|
+
});
|
|
121
|
+
pending = pending.filter((index) => !results[index]);
|
|
122
|
+
if (pending.length)
|
|
123
|
+
await (0, runtime_js_1.sleep)((0, runtime_js_1.backoff)(++polls), options.signal);
|
|
124
|
+
}
|
|
125
|
+
return results;
|
|
126
|
+
}
|
|
127
|
+
finally {
|
|
128
|
+
cache.close();
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
//# sourceMappingURL=batch.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"batch.js","sourceRoot":"","sources":["../../src/batch.ts"],"names":[],"mappings":";;;AAAA,2CAA2E;AAC3E,yCAAyE;AAEzE,2CAAqC;AACrC,6CAQsB;AAUf,KAAK,mBAAmB,KAA4B,EAAE,OAAqB;IAChF,OAAO,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC;IACjC,MAAM,KAAK,GAAG,IAAI,mBAAQ,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IAC9C,MAAM,IAAI,GAAG,IAAA,oBAAS,EAAC,KAAK,EAAE,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;IAC/D,MAAM,KAAK,GAAG,OAAO,CAAC,WAAW,IAAI,EAAE,CAAC;IACxC,MAAM,UAAU,GAAG,CAAC,IAAI,sBAAS,CAAC,KAAK,CAAC,CAAC,CAAC;IAC1C,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;IAClC,MAAM,QAAQ,GAAG,IAAA,gBAAK,GAAE,CAAC;IACzB,MAAM,OAAO,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;IACvC,MAAM,OAAO,GAAa,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IAElD,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,KAAK,CAAC,WAAW,CAAC,IAAI,EAAE,OAAO,CAAC,QAAQ,EAAE,QAAQ,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;QAClF,MAAM,QAAQ,GAAG,CAAC,KAAa,EAAE,EAAE,CACjC,IAAA,+BAAkB,EAAC,mBAAmB,kBAAkB,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,GAAG,OAAO,EAAE,UAAU,EAAE,CAAC,CAAC;QACjG,MAAM,OAAO,GAAG,CAAC,KAAa,EAAE,EAAE,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,KAAK,CAAE,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAE,CAAC;QAExG,MAAM,MAAM,GAAG,KAAK,EAAE,KAAa,EAAE,cAAc,GAAG,KAAK,EAAuB,EAAE;YAClF,IAAI,KAAK,GAAG,cAAc,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAE,CAAC;YAC9D,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;gBACjB,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;gBAC7C,6EAA6E;gBAC7E,IACE,CAAC,CAAC,OAAO,CAAC,KAAK,IAAI,OAAO,CAAC,IAAI,EAAE,KAAK,KAAK,SAAS,CAAC;oBACrD,CAAC,OAAO,CAAC,KAAK,IAAI,CAAC,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,UAAU,IAAI,CAAC,CAAC,CAAC;oBAEhE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;gBAC5B,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC;YACzB,CAAC;YACD,MAAM,OAAO,GAAG,MAAM,IAAA,+BAAkB,EAAC,KAAK,CAAC,KAAK,CAAE,CAAC,IAAI,GAAG,QAAQ,EAAE;gBACtE,GAAG,OAAO;gBACV,UAAU;gBACV,MAAM,EAAE,MAAM;gBACd,IAAI,EAAE,KAAK,CAAC,KAAK,CAAE,CAAC,WAAW,EAAE;gBACjC,gBAAgB,EAAE,CAAC,GAAG,EAAE,GAAG,CAAC;gBAC5B,OAAO,EAAE;oBACP,iBAAiB,EAAE,KAAK,CAAC,UAAU;oBACnC,uBAAuB,EAAE,KAAK,CAAC,SAAS;oBACxC,yBAAyB,EAAE,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC;iBACpD;aACF,CAAC,CAAC;YACH,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;gBACnB,MAAM,GAAG,GAAG,OAAO,CAAC,IAAK,CAAC;gBAC1B,IAAI,OAAO,GAAG,CAAC,MAAM,KAAK,QAAQ,IAAI,CAAC,GAAG,CAAC,MAAM;oBAC/C,OAAO,CAAC,KAAK,GAAG,IAAI,0BAAc,CAAC,sCAAsC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;qBAC7F,CAAC;oBACJ,KAAK,CAAC,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC;oBAC1B,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAE,EAAE,KAAK,CAAC,UAAU,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,UAAU,CAAC,CAAC;oBACvE,IAAI,GAAG,CAAC,KAAK,KAAK,SAAS,IAAI,OAAO,CAAC,QAAQ,IAAI,CAAC,cAAc;wBAAE,OAAO,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;gBACjG,CAAC;YACH,CAAC;YACD,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;QAC5B,CAAC,CAAC;QAEF,MAAM,WAAW,GAAiB,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAC1D,MAAM,IAAA,iBAAI,EAAC,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE;YACzC,WAAW,CAAC,KAAK,CAAC,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,CAAC;QAC3C,CAAC,CAAC,CAAC;QAEH,MAAM,OAAO,GAAG,CAAC,KAAa,EAAE,EAAE,KAAK,EAAE,OAAO,EAAc,EAAiB,EAAE;YAC/E,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;gBACnB,MAAM,GAAG,GAAG,OAAO,CAAC,IAAK,CAAC;gBAC1B,IAAI,OAAO,GAAG,CAAC,MAAM,KAAK,QAAQ,IAAI,CAAC,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC;oBACjF,OAAO,CAAC,KAAK,GAAG,IAAI,0BAAc,CAAC,uCAAuC,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;gBACpG,CAAC;qBAAM,IAAI,GAAG,CAAC,KAAK,KAAK,WAAW,IAAI,GAAG,CAAC,KAAK,KAAK,QAAQ,EAAE,CAAC;oBAC/D,MAAM,MAAM,GAAG,GAAG,CAAC,WAAW,CAAC;oBAC/B,MAAM,OAAO,GAAG,GAAG,CAAC,QAAQ,CAAC;oBAC7B,IACE,OAAO,MAAM,KAAK,QAAQ;wBAC1B,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC;wBACzB,MAAM,GAAG,GAAG;wBACZ,MAAM,GAAG,GAAG;wBACZ,CAAC,IAAA,oBAAQ,EAAC,OAAO,CAAC;wBAClB,CAAC,GAAG,CAAC,KAAK,KAAK,WAAW,CAAC,KAAK,CAAC,MAAM,KAAK,GAAG,CAAC,EAChD,CAAC;wBACD,OAAO,CAAC,KAAK,GAAG,IAAI,0BAAc,CAAC,yCAAyC,EAAE;4BAC5E,SAAS,EAAE,KAAK;yBACjB,CAAC,CAAC;oBACL,CAAC;yBAAM,CAAC;wBACN,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAE,EAAE,KAAK,CAAC,UAAU,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,UAAU,CAAC,CAAC;wBACvE,OAAO,IAAI,kBAAM,CACf,KAAK,EACL,KAAK,CAAC,KAAK,CAAE,EACb;4BACE,IAAI,EAAE,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI;4BACrC,KAAK,EAAE,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAA,oBAAQ,EAAC,MAAM,EAAE,OAAO,CAAC;4BACxD,UAAU,EAAE,MAAM;4BAClB,SAAS,EAAE,GAAG,CAAC,MAAM;4BACrB,QAAQ,EAAE,OAAO,CAAC,QAAQ;yBAC3B,EACD,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,GAAG,IAAI,EACpC,GAAG,CAAC,MAAM,CACX,CAAC;oBACJ,CAAC;gBACH,CAAC;qBAAM,IAAI,GAAG,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;oBACnC,OAAO,CAAC,KAAK,GAAG,IAAI,oBAAQ,CAAC,sCAAsC,EAAE;wBACnE,UAAU,EAAE,GAAG;wBACf,SAAS,EAAE,aAAa;wBACxB,SAAS,EAAE,KAAK,CAAC,MAAM,IAAI,EAAE;qBAC9B,CAAC,CAAC;oBACH,OAAO,CAAC,UAAU,GAAG,GAAG,CAAC;gBAC3B,CAAC;qBAAM,IAAI,GAAG,CAAC,KAAK,KAAK,QAAQ,IAAI,GAAG,CAAC,KAAK,KAAK,SAAS;oBAAE,OAAO,IAAI,CAAC;;oBACrE,OAAO,CAAC,KAAK,GAAG,IAAI,0BAAc,CAAC,sCAAsC,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;YACxG,CAAC;YACD,OAAO,IAAI,kBAAM,CACf,KAAK,EACL,KAAK,CAAC,KAAK,CAAE,EACb,EAAE,GAAG,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,CAAC,MAAM,IAAI,OAAO,CAAC,SAAS,EAAE,EACxE,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,GAAG,IAAI,EACpC,KAAK,CAAC,MAAM,IAAI,EAAE,CACnB,CAAC;QACJ,CAAC,CAAC;QAEF,IAAI,OAAO,GAAG,OAAO,CAAC;QACtB,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,OAAO,OAAO,CAAC,MAAM,EAAE,CAAC;YACtB,MAAM,IAAA,iBAAI,EAAC,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE;gBACzC,IAAI,MAAM,GAAG,OAAO,CAAC,KAAK,EAAE,WAAW,CAAC,KAAK,CAAE,CAAC,CAAC;gBACjD,IAAI,CAAC,MAAM,EAAE,CAAC;oBACZ,MAAM,KAAK,GAAG,WAAW,CAAC,KAAK,CAAE,CAAC,KAAK,CAAC;oBACxC,WAAW,CAAC,KAAK,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,QAAQ,CAAC,KAAK,CAAC,MAAO,CAAC,EAAE,CAAC;oBACvE,MAAM,GAAG,OAAO,CAAC,KAAK,EAAE,WAAW,CAAC,KAAK,CAAE,CAAC,CAAC;gBAC/C,CAAC;gBACD,IAAI,MAAM,EAAE,CAAC;oBACX,OAAO,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC;oBACxB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;gBAC3B,CAAC;YACH,CAAC,CAAC,CAAC;YACH,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;YACrD,IAAI,OAAO,CAAC,MAAM;gBAAE,MAAM,IAAA,kBAAK,EAAC,IAAA,oBAAO,EAAC,EAAE,KAAK,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;QACpE,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;YAAS,CAAC;QACT,KAAK,CAAC,KAAK,EAAE,CAAC;IAChB,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import type { AnyRequest } from "./models.js";
|
|
2
|
+
export interface CacheEntry {
|
|
3
|
+
submission: string;
|
|
4
|
+
job_id: string | null;
|
|
5
|
+
batch_key: string;
|
|
6
|
+
batch_size: number;
|
|
7
|
+
}
|
|
8
|
+
export declare const newId: () => string;
|
|
9
|
+
/** Fingerprints include the key, origin, all parameters, and duplicate occurrence. */
|
|
10
|
+
export declare function cacheKeys(items: readonly AnyRequest[], apiKey: string, baseUrl: string): string[];
|
|
11
|
+
/** SQLite FULL commits persist the entire workload before its first POST. */
|
|
12
|
+
export declare class JobCache {
|
|
13
|
+
readonly path: string;
|
|
14
|
+
private readonly db;
|
|
15
|
+
constructor(path?: string);
|
|
16
|
+
private transaction;
|
|
17
|
+
prepareMany(keys: readonly string[], useCache: boolean, batchKey: string, batchSize: number): CacheEntry[];
|
|
18
|
+
save(key: string, submission: string, jobId: string, expiresAt?: unknown): void;
|
|
19
|
+
close(): void;
|
|
20
|
+
}
|
|
21
|
+
//# sourceMappingURL=cache.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cache.d.ts","sourceRoot":"","sources":["../../src/cache.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAE9C,MAAM,WAAW,UAAU;IACzB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;CACpB;AACD,eAAO,MAAM,KAAK,QAAO,MAA0C,CAAC;AAGpE,sFAAsF;AACtF,wBAAgB,SAAS,CAAC,KAAK,EAAE,SAAS,UAAU,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,CAejG;AAED,6EAA6E;AAC7E,qBAAa,QAAQ;IACnB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAe;IAElC,YAAY,IAAI,CAAC,EAAE,MAAM,EAyBxB;IAED,OAAO,CAAC,WAAW;IAYnB,WAAW,CAAC,IAAI,EAAE,SAAS,MAAM,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,UAAU,EAAE,CAgCzG;IAED,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,OAAO,GAAG,IAAI,CAK9E;IAED,KAAK,IAAI,IAAI,CAEZ;CACF"}
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.JobCache = exports.newId = void 0;
|
|
4
|
+
exports.cacheKeys = cacheKeys;
|
|
5
|
+
const node_crypto_1 = require("node:crypto");
|
|
6
|
+
const node_fs_1 = require("node:fs");
|
|
7
|
+
const node_os_1 = require("node:os");
|
|
8
|
+
const node_path_1 = require("node:path");
|
|
9
|
+
const node_sqlite_1 = require("node:sqlite");
|
|
10
|
+
const newId = () => (0, node_crypto_1.randomUUID)().replaceAll("-", "");
|
|
11
|
+
exports.newId = newId;
|
|
12
|
+
const hash = (value) => (0, node_crypto_1.createHash)("sha256").update(value).digest("hex");
|
|
13
|
+
/** Fingerprints include the key, origin, all parameters, and duplicate occurrence. */
|
|
14
|
+
function cacheKeys(items, apiKey, baseUrl) {
|
|
15
|
+
const identity = hash(apiKey);
|
|
16
|
+
const occurrences = new Map();
|
|
17
|
+
return items.map((item) => {
|
|
18
|
+
const params = item.queryParams();
|
|
19
|
+
const sorted = Object.fromEntries(Object.keys(params)
|
|
20
|
+
.sort()
|
|
21
|
+
.map((key) => [key, params[key]]));
|
|
22
|
+
const canonical = JSON.stringify([identity, baseUrl, item.path, sorted]);
|
|
23
|
+
const occurrence = occurrences.get(canonical) ?? 0;
|
|
24
|
+
occurrences.set(canonical, occurrence + 1);
|
|
25
|
+
return hash(`${canonical}:${occurrence}`);
|
|
26
|
+
});
|
|
27
|
+
}
|
|
28
|
+
/** SQLite FULL commits persist the entire workload before its first POST. */
|
|
29
|
+
class JobCache {
|
|
30
|
+
path;
|
|
31
|
+
db;
|
|
32
|
+
constructor(path) {
|
|
33
|
+
this.path = (0, node_path_1.resolve)(path || process.env.LITESCRAPE_JOB_CACHE || (0, node_path_1.join)((0, node_os_1.homedir)(), ".cache", "litescrape", "jobs-ts.sqlite3"));
|
|
34
|
+
(0, node_fs_1.mkdirSync)((0, node_path_1.dirname)(this.path), { recursive: true, mode: 0o700 });
|
|
35
|
+
this.db = new node_sqlite_1.DatabaseSync(this.path);
|
|
36
|
+
try {
|
|
37
|
+
if (process.platform !== "win32")
|
|
38
|
+
(0, node_fs_1.chmodSync)(this.path, 0o600);
|
|
39
|
+
this.db.exec(`PRAGMA busy_timeout=30000; PRAGMA journal_mode=WAL; PRAGMA synchronous=FULL;
|
|
40
|
+
CREATE TABLE IF NOT EXISTS jobs (
|
|
41
|
+
cache_key TEXT PRIMARY KEY, submission TEXT NOT NULL, job_id TEXT,
|
|
42
|
+
expires_at TEXT, updated_at REAL NOT NULL
|
|
43
|
+
);`);
|
|
44
|
+
this.transaction(() => {
|
|
45
|
+
const columns = this.db
|
|
46
|
+
.prepare("PRAGMA table_info(jobs)")
|
|
47
|
+
.all()
|
|
48
|
+
.map((row) => row.name);
|
|
49
|
+
if (!columns.includes("batch_key"))
|
|
50
|
+
this.db.exec("ALTER TABLE jobs ADD COLUMN batch_key TEXT");
|
|
51
|
+
if (!columns.includes("batch_size"))
|
|
52
|
+
this.db.exec("ALTER TABLE jobs ADD COLUMN batch_size INTEGER");
|
|
53
|
+
});
|
|
54
|
+
}
|
|
55
|
+
catch (error) {
|
|
56
|
+
this.db.close();
|
|
57
|
+
throw error;
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
transaction(operation) {
|
|
61
|
+
this.db.exec("BEGIN IMMEDIATE");
|
|
62
|
+
try {
|
|
63
|
+
const result = operation();
|
|
64
|
+
this.db.exec("COMMIT");
|
|
65
|
+
return result;
|
|
66
|
+
}
|
|
67
|
+
catch (error) {
|
|
68
|
+
this.db.exec("ROLLBACK");
|
|
69
|
+
throw error;
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
prepareMany(keys, useCache, batchKey, batchSize) {
|
|
73
|
+
return this.transaction(() => keys.map((key) => {
|
|
74
|
+
const row = this.db
|
|
75
|
+
.prepare("SELECT submission,job_id,batch_key,batch_size FROM jobs WHERE cache_key=?")
|
|
76
|
+
.get(key);
|
|
77
|
+
if (useCache && row) {
|
|
78
|
+
if (!row.batch_key) {
|
|
79
|
+
row.batch_key = batchKey;
|
|
80
|
+
row.batch_size = batchSize;
|
|
81
|
+
this.db
|
|
82
|
+
.prepare("UPDATE jobs SET batch_key=?,batch_size=? WHERE cache_key=?")
|
|
83
|
+
.run(batchKey, batchSize, key);
|
|
84
|
+
}
|
|
85
|
+
return row;
|
|
86
|
+
}
|
|
87
|
+
const entry = {
|
|
88
|
+
submission: (0, exports.newId)(),
|
|
89
|
+
job_id: null,
|
|
90
|
+
batch_key: batchKey,
|
|
91
|
+
batch_size: batchSize,
|
|
92
|
+
};
|
|
93
|
+
this.db
|
|
94
|
+
.prepare(`INSERT INTO jobs(cache_key,submission,updated_at,batch_key,batch_size) VALUES (?,?,?,?,?)
|
|
95
|
+
ON CONFLICT(cache_key) DO UPDATE SET submission=excluded.submission,job_id=NULL,expires_at=NULL,
|
|
96
|
+
updated_at=excluded.updated_at,batch_key=excluded.batch_key,batch_size=excluded.batch_size`)
|
|
97
|
+
.run(key, entry.submission, Date.now() / 1000, batchKey, batchSize);
|
|
98
|
+
return entry;
|
|
99
|
+
}));
|
|
100
|
+
}
|
|
101
|
+
save(key, submission, jobId, expiresAt) {
|
|
102
|
+
// A concurrent fresh run can replace a slot; its submission must not be overwritten.
|
|
103
|
+
this.db
|
|
104
|
+
.prepare("UPDATE jobs SET job_id=?,expires_at=?,updated_at=? WHERE cache_key=? AND submission=?")
|
|
105
|
+
.run(jobId, typeof expiresAt === "string" ? expiresAt : null, Date.now() / 1000, key, submission);
|
|
106
|
+
}
|
|
107
|
+
close() {
|
|
108
|
+
this.db.close();
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
exports.JobCache = JobCache;
|
|
112
|
+
//# sourceMappingURL=cache.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cache.js","sourceRoot":"","sources":["../../src/cache.ts"],"names":[],"mappings":";;;;AAAA,6CAAqD;AACrD,qCAA+C;AAC/C,qCAAkC;AAClC,yCAAmD;AACnD,6CAA2C;AASpC,MAAM,KAAK,GAAG,GAAW,EAAE,CAAC,IAAA,wBAAU,GAAE,CAAC,UAAU,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;AAAvD,QAAA,KAAK,GAAL,KAAK,CAAkD;AACpE,MAAM,IAAI,GAAG,CAAC,KAAa,EAAE,EAAE,CAAC,IAAA,wBAAU,EAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAEjF,sFAAsF;AACtF,mBAA0B,KAA4B,EAAE,MAAc,EAAE,OAAe;IACrF,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC;IAC9B,MAAM,WAAW,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC9C,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACxB,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;QAClC,MAAM,MAAM,GAAG,MAAM,CAAC,WAAW,CAC/B,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC;aAChB,IAAI,EAAE;aACN,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CACpC,CAAC;QACF,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,QAAQ,EAAE,OAAO,EAAE,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;QACzE,MAAM,UAAU,GAAG,WAAW,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QACnD,WAAW,CAAC,GAAG,CAAC,SAAS,EAAE,UAAU,GAAG,CAAC,CAAC,CAAC;QAC3C,OAAO,IAAI,CAAC,GAAG,SAAS,IAAI,UAAU,EAAE,CAAC,CAAC;IAC5C,CAAC,CAAC,CAAC;AACL,CAAC;AAED,6EAA6E;AAC7E;IACW,IAAI,CAAS;IACL,EAAE,CAAe;IAElC,YAAY,IAAa;QACvB,IAAI,CAAC,IAAI,GAAG,IAAA,mBAAO,EACjB,IAAI,IAAI,OAAO,CAAC,GAAG,CAAC,oBAAoB,IAAI,IAAA,gBAAI,EAAC,IAAA,iBAAO,GAAE,EAAE,QAAQ,EAAE,YAAY,EAAE,iBAAiB,CAAC,CACvG,CAAC;QACF,IAAA,mBAAS,EAAC,IAAA,mBAAO,EAAC,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QAChE,IAAI,CAAC,EAAE,GAAG,IAAI,0BAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACtC,IAAI,CAAC;YACH,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO;gBAAE,IAAA,mBAAS,EAAC,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;YAC9D,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC;;;;WAIR,CAAC,CAAC;YACP,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE;gBACpB,MAAM,OAAO,GAAG,IAAI,CAAC,EAAE;qBACpB,OAAO,CAAC,yBAAyB,CAAC;qBAClC,GAAG,EAAE;qBACL,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBAC1B,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC;oBAAE,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,4CAA4C,CAAC,CAAC;gBAC/F,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC;oBAAE,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,gDAAgD,CAAC,CAAC;YACtG,CAAC,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC;YAChB,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAEO,WAAW,CAAI,SAAkB;QACvC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;QAChC,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;YAC3B,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACvB,OAAO,MAAM,CAAC;QAChB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YACzB,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED,WAAW,CAAC,IAAuB,EAAE,QAAiB,EAAE,QAAgB,EAAE,SAAiB;QACzF,OAAO,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,CAC3B,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;YACf,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE;iBAChB,OAAO,CAAC,2EAA2E,CAAC;iBACpF,GAAG,CAAC,GAAG,CAAC,CAAC;YACZ,IAAI,QAAQ,IAAI,GAAG,EAAE,CAAC;gBACpB,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,CAAC;oBACnB,GAAG,CAAC,SAAS,GAAG,QAAQ,CAAC;oBACzB,GAAG,CAAC,UAAU,GAAG,SAAS,CAAC;oBAC3B,IAAI,CAAC,EAAE;yBACJ,OAAO,CAAC,4DAA4D,CAAC;yBACrE,GAAG,CAAC,QAAQ,EAAE,SAAS,EAAE,GAAG,CAAC,CAAC;gBACnC,CAAC;gBACD,OAAO,GAA4B,CAAC;YACtC,CAAC;YACD,MAAM,KAAK,GAAe;gBACxB,UAAU,EAAE,IAAA,QAAA,KAAK,GAAE;gBACnB,MAAM,EAAE,IAAI;gBACZ,SAAS,EAAE,QAAQ;gBACnB,UAAU,EAAE,SAAS;aACtB,CAAC;YACF,IAAI,CAAC,EAAE;iBACJ,OAAO,CACN;;qGAEyF,CAC1F;iBACA,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,EAAE,QAAQ,EAAE,SAAS,CAAC,CAAC;YACtE,OAAO,KAAK,CAAC;QACf,CAAC,CAAC,CACH,CAAC;IACJ,CAAC;IAED,IAAI,CAAC,GAAW,EAAE,UAAkB,EAAE,KAAa,EAAE,SAAmB;QACtE,qFAAqF;QACrF,IAAI,CAAC,EAAE;aACJ,OAAO,CAAC,uFAAuF,CAAC;aAChG,GAAG,CAAC,KAAK,EAAE,OAAO,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,EAAE,GAAG,EAAE,UAAU,CAAC,CAAC;IACtG,CAAC;IAED,KAAK;QACH,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC;IAClB,CAAC;CACF"}
|