@opencraw/driver 0.7.10 → 0.7.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -77
- package/package.json +4 -45
- package/dist/index.esm.js +0 -655
- package/dist/main.esm.js +0 -563
- package/dist/src/callout-bridge/client-hooks.client.d.ts +0 -47
- package/dist/src/callout-bridge/index.d.ts +0 -3
- package/dist/src/index.d.ts +0 -16
- package/dist/src/main.d.ts +0 -3
- package/dist/src/protocol/driver-json-schema.mapper.d.ts +0 -9
- package/dist/src/protocol/driver-message.contract.d.ts +0 -492
- package/dist/src/protocol/index.d.ts +0 -4
- package/dist/src/record-files/diff-record-files.use-case.d.ts +0 -10
- package/dist/src/record-files/export-record-files.use-case.d.ts +0 -11
- package/dist/src/record-files/index.d.ts +0 -3
- package/dist/src/runtime-manifest/index.d.ts +0 -4
- package/dist/src/runtime-manifest/runtime-manifest-json-schema.mapper.d.ts +0 -8
- package/dist/src/runtime-manifest/runtime-manifest.contract.d.ts +0 -28
- package/dist/src/server/index.d.ts +0 -3
- package/dist/src/server/serve-driver.use-case.d.ts +0 -17
- package/dist/src/session/index.d.ts +0 -5
- package/dist/src/session/recipe-loading.use-case.d.ts +0 -23
- package/dist/src/session/run-session.use-case.d.ts +0 -35
- package/dist/src/session/streaming-sink.repository.d.ts +0 -8
- package/dist/src/site-access/index.d.ts +0 -3
- package/dist/src/site-access/resolve-site-access.use-case.d.ts +0 -26
- package/dist/src/site-probe/index.d.ts +0 -2
- package/dist/src/site-probe/probe-site.use-case.d.ts +0 -13
- package/dist/src/transport/index.d.ts +0 -5
- package/dist/src/transport/json-rpc.contract.d.ts +0 -31
- package/dist/src/transport/line-channel.client.d.ts +0 -27
- package/dist/src/work-bridge/client-work-source.client.d.ts +0 -17
- package/dist/src/work-bridge/index.d.ts +0 -2
- package/schemas/driver-protocol.schema.json +0 -1104
- package/schemas/driver-runtime.fixture.json +0 -9
- package/schemas/driver-runtime.schema.json +0 -63
package/README.md
CHANGED
|
@@ -1,79 +1,3 @@
|
|
|
1
|
-
<p align="center">
|
|
2
|
-
<img src="https://raw.githubusercontent.com/Open-Craw/OpenCraw/main/docs/assets/opencraw-logo.svg" alt="OpenCraw" width="360">
|
|
3
|
-
</p>
|
|
4
|
-
|
|
5
1
|
# @opencraw/driver
|
|
6
2
|
|
|
7
|
-
|
|
8
|
-
to. A client in Python, Go, Dart, C#, Rust, Java, Ruby, PHP or C++ starts `opencraw-driver` once, and speaks
|
|
9
|
-
**JSON-RPC 2.0 over its stdin and stdout, one message per line**. It sends recipes, gets every crawl event and
|
|
10
|
-
record as it happens, and answers the recipes' hooks with functions in its own language. This is how the
|
|
11
|
-
OpenCraw packages for other languages work (`docs/research/driver-and-language-packages.md`).
|
|
12
|
-
|
|
13
|
-
```sh
|
|
14
|
-
npx opencraw-driver # then write JSON-RPC lines to it
|
|
15
|
-
```
|
|
16
|
-
|
|
17
|
-
## The conversation
|
|
18
|
-
|
|
19
|
-
```jsonc
|
|
20
|
-
→ {"jsonrpc":"2.0","id":1,"method":"initialize","params":{"clientName":"opencraw-python","clientVersion":"1.0.0","protocolVersion":"1.2.0"}}
|
|
21
|
-
← {"jsonrpc":"2.0","id":1,"result":{"driverVersion":"0.2.0","protocolVersion":"1.2.0","methods":["initialize","validate","run","work","cancel","probe","diff","export","shutdown"]}}
|
|
22
|
-
→ {"jsonrpc":"2.0","id":2,"method":"run","params":{"paths":["recipes/"],"hooks":["tidy"]}}
|
|
23
|
-
← {"jsonrpc":"2.0","id":2,"result":{"runId":"run-1"}}
|
|
24
|
-
← {"jsonrpc":"2.0","method":"event","params":{"runId":"run-1","event":{"type":"recipe:start",…}}}
|
|
25
|
-
← {"jsonrpc":"2.0","id":1,"method":"callout","params":{"runId":"run-1","request":{"kind":"hook","name":"tidy","input":" A ",…}}}
|
|
26
|
-
→ {"jsonrpc":"2.0","id":1,"result":{"status":"ok","output":"a"}}
|
|
27
|
-
← {"jsonrpc":"2.0","method":"record","params":{"runId":"run-1","recipeId":"things","key":"…","data":{…}}}
|
|
28
|
-
← {"jsonrpc":"2.0","method":"finished","params":{"runId":"run-1","report":{…}}}
|
|
29
|
-
→ {"jsonrpc":"2.0","id":3,"method":"shutdown"}
|
|
30
|
-
```
|
|
31
|
-
|
|
32
|
-
| Method | From | Does |
|
|
33
|
-
|---|---|---|
|
|
34
|
-
| `initialize` | client | Must come first. The driver refuses a protocol of another major. |
|
|
35
|
-
| `validate` | client | Loads recipes (`paths` or inline `recipes`) without running them: the inputs that would run, every hook they call, and every problem with its path. |
|
|
36
|
-
| `run` | client | Starts a run and answers its `runId` at once; the crawl goes on in the background. `hooks`, `captchaSolvers` and `accessPlugins` name what the client answers; `accessConfig` and `access` pick the access config and profile; `out` writes records to a file instead of sending them. |
|
|
37
|
-
| `work` | client | Starts a worker pool and answers its `runId` at once. Its windows ask the client for items (`work.next`) and keep the steps marked `keep` between them; `windows` is a count or core's `WindowsPolicy`. Takes the same crawler fields as `run`. |
|
|
38
|
-
| `cancel` | client | Stops a run (its crawler is closed); it still ends with `finished`. |
|
|
39
|
-
| `probe` | client | Fetches a page (or renders it, with `browser`) and says where its data lives: the CLI's and MCP's probe, through the same access fields as `run`. |
|
|
40
|
-
| `diff` | client | Compares two runs' JSON Lines files by key: counts, the report a person reads, and the changes up to `limit`. |
|
|
41
|
-
| `export` | client | Writes a run's JSON Lines file in another format (the extension of `out`), with an output recipe's columns. |
|
|
42
|
-
| `shutdown` | client | Closes every run, answers, then exits. Closing stdin does the same. |
|
|
43
|
-
| `event` | driver | Every crawl event, as core reports it. |
|
|
44
|
-
| `record` | driver | Each record, unless the run has `out`. |
|
|
45
|
-
| `finished` | driver | The run is over: its report (for `work`, the work report), or why it could not run. |
|
|
46
|
-
| `work.next` | driver | A window wants an item: answer `{ "item": { "id", "vars", "recipe"? } }`, holding the answer until there is one, or `{ "item": null }` to end the work. An id already queued or running runs once. |
|
|
47
|
-
| `work.done`, `work.failed` | driver | How an item ended: its report and records, or its outcome (`failure`, `neutral`). |
|
|
48
|
-
| `callout` | driver | A hook, captcha solver or access plugin the client serves (`request.kind`: `hook`, `captcha`, `access`). The request and its answer are core's [callout contract](../core/README.md): `ok`, `error`, or `pending` with `retryAfterMs`, asked again with the same idempotency key. |
|
|
49
|
-
|
|
50
|
-
A run's answer (`run`, `work`) always comes before any of its events, records or callouts. Every answer comes back before the driver stops, even when a client writes its last requests and closes
|
|
51
|
-
stdin at once. Answers may arrive out of order; match them by `id`.
|
|
52
|
-
|
|
53
|
-
## For client authors
|
|
54
|
-
|
|
55
|
-
- `schemas/driver-protocol.schema.json` is every message's shape, generated from the protocol's zod
|
|
56
|
-
contracts. Generate your models from it rather than typing them by hand.
|
|
57
|
-
- `conformance/*.json` are recorded conversations. A client is done when it replays them all against the real
|
|
58
|
-
driver. See `conformance/README.md`.
|
|
59
|
-
- Errors are JSON-RPC's codes: `-32700` (not JSON), `-32600` (not JSON-RPC), `-32601` (no such method),
|
|
60
|
-
`-32602` (bad params, the problems listed), `-32000` (a failure to act on: its message says what to do).
|
|
61
|
-
- stdout carries protocol messages only. The driver's own logs go to stderr.
|
|
62
|
-
|
|
63
|
-
## The runtime: no Node needed
|
|
64
|
-
|
|
65
|
-
Each driver release carries, as assets of its GitHub release `driver@<version>`, an archive per platform
|
|
66
|
-
(`win32-x64`, `darwin-arm64`, `darwin-x64`, `linux-x64`, `linux-arm64`): a Node binary, the driver and its
|
|
67
|
-
dependencies, and `runtime.json`, whose `command` starts it (both paths relative to the unpacked folder).
|
|
68
|
-
`driver-runtime.json` lists them with their SHA-256 and size; its schema is `schemas/driver-runtime.schema.json`
|
|
69
|
-
and `schemas/driver-runtime.fixture.json` is an example. A client downloads its platform's archive once, refuses
|
|
70
|
-
one whose hash does not match, unpacks it into a per-user cache and starts the driver from there.
|
|
71
|
-
|
|
72
|
-
`node packages/driver/packaging/stage-runtime.mjs` builds this platform's archive into `dist/drop/driver-runtime`
|
|
73
|
-
(after `smith build -p driver`), and starts the staged driver once with `PATH` emptied, to prove it needs nothing
|
|
74
|
-
else. Browsers are not in it: they are installed through the driver, the Playwright way.
|
|
75
|
-
|
|
76
|
-
## Development
|
|
77
|
-
|
|
78
|
-
`smith run lint typecheck test build -p driver`. After changing a contract in `src/protocol/`, run
|
|
79
|
-
`smith run schemas -p driver` to rewrite `schemas/driver-protocol.schema.json`; a test fails until you do.
|
|
3
|
+
This package is no longer available.
|
package/package.json
CHANGED
|
@@ -1,47 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@opencraw/driver",
|
|
3
|
-
"version": "0.7.
|
|
4
|
-
"description": "
|
|
5
|
-
"license": "MIT"
|
|
6
|
-
|
|
7
|
-
"smith": {
|
|
8
|
-
"kind": "package",
|
|
9
|
-
"stack": "node-vitest"
|
|
10
|
-
},
|
|
11
|
-
"main": "./dist/index.esm.js",
|
|
12
|
-
"module": "./dist/index.esm.js",
|
|
13
|
-
"types": "./dist/src/index.d.ts",
|
|
14
|
-
"exports": {
|
|
15
|
-
"./package.json": "./package.json",
|
|
16
|
-
".": {
|
|
17
|
-
"types": "./dist/src/index.d.ts",
|
|
18
|
-
"import": "./dist/index.esm.js",
|
|
19
|
-
"default": "./dist/index.esm.js"
|
|
20
|
-
}
|
|
21
|
-
},
|
|
22
|
-
"bin": {
|
|
23
|
-
"opencraw-driver": "./dist/main.esm.js"
|
|
24
|
-
},
|
|
25
|
-
"files": [
|
|
26
|
-
"dist",
|
|
27
|
-
"schemas",
|
|
28
|
-
"!**/*.tsbuildinfo",
|
|
29
|
-
"!**/*.d.ts.map",
|
|
30
|
-
"!**/*.js.map"
|
|
31
|
-
],
|
|
32
|
-
"publishConfig": {
|
|
33
|
-
"access": "public"
|
|
34
|
-
},
|
|
35
|
-
"scripts": {
|
|
36
|
-
"build": "esbuild src/index.ts src/main.ts --bundle --packages=external --platform=node --format=esm --outdir=dist --out-extension:.js=.esm.js --sourcemap && tsc -p tsconfig.build.json && node ../../declaration-specifiers.mjs dist/src",
|
|
37
|
-
"typecheck": "tsc -p tsconfig.json",
|
|
38
|
-
"lint": "eslint .",
|
|
39
|
-
"test": "vitest run",
|
|
40
|
-
"schemas": "node tools/emit-protocol-schema.mjs"
|
|
41
|
-
},
|
|
42
|
-
"dependencies": {
|
|
43
|
-
"@opencraw/cli": "^0.2.2",
|
|
44
|
-
"@opencraw/core": "^0.5.0",
|
|
45
|
-
"zod": "^4.6.5"
|
|
46
|
-
}
|
|
47
|
-
}
|
|
3
|
+
"version": "0.7.11",
|
|
4
|
+
"description": "This package is no longer available.",
|
|
5
|
+
"license": "MIT"
|
|
6
|
+
}
|