@getopencrawl/sdk 0.0.0-stage → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +598 -2
- package/dist/agentPlatform.d.ts +142 -0
- package/dist/agentPlatform.d.ts.map +1 -0
- package/dist/agentPlatform.js +46 -0
- package/dist/agentPlatform.js.map +1 -0
- package/dist/agents.d.ts +237 -0
- package/dist/agents.d.ts.map +1 -0
- package/dist/agents.js +279 -0
- package/dist/agents.js.map +1 -0
- package/dist/answerContract.d.ts +85 -0
- package/dist/answerContract.d.ts.map +1 -0
- package/dist/answerContract.js +112 -0
- package/dist/answerContract.js.map +1 -0
- package/dist/answerView.d.ts +40 -0
- package/dist/answerView.d.ts.map +1 -0
- package/dist/answerView.js +118 -0
- package/dist/answerView.js.map +1 -0
- package/dist/configuration.d.ts +5 -0
- package/dist/configuration.d.ts.map +1 -0
- package/dist/configuration.js +29 -0
- package/dist/configuration.js.map +1 -0
- package/dist/customers.d.ts +176 -0
- package/dist/customers.d.ts.map +1 -0
- package/dist/customers.js +286 -0
- package/dist/customers.js.map +1 -0
- package/dist/errors.d.ts +35 -0
- package/dist/errors.d.ts.map +1 -0
- package/dist/errors.js +58 -0
- package/dist/errors.js.map +1 -0
- package/dist/eventEvidence.d.ts +10 -0
- package/dist/eventEvidence.d.ts.map +1 -0
- package/dist/eventEvidence.js +52 -0
- package/dist/eventEvidence.js.map +1 -0
- package/dist/index.d.ts +98 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +139 -0
- package/dist/index.js.map +1 -0
- package/dist/productMonitors.d.ts +258 -0
- package/dist/productMonitors.d.ts.map +1 -0
- package/dist/productMonitors.js +68 -0
- package/dist/productMonitors.js.map +1 -0
- package/dist/researchAnswerContract.d.ts +8 -0
- package/dist/researchAnswerContract.d.ts.map +1 -0
- package/dist/researchAnswerContract.js +23 -0
- package/dist/researchAnswerContract.js.map +1 -0
- package/dist/researchAnswerView.d.ts +8 -0
- package/dist/researchAnswerView.d.ts.map +1 -0
- package/dist/researchAnswerView.js +27 -0
- package/dist/researchAnswerView.js.map +1 -0
- package/dist/researchCompletion.d.ts +114 -0
- package/dist/researchCompletion.d.ts.map +1 -0
- package/dist/researchCompletion.js +94 -0
- package/dist/researchCompletion.js.map +1 -0
- package/dist/serviceTenants.d.ts +15 -0
- package/dist/serviceTenants.d.ts.map +1 -0
- package/dist/serviceTenants.js +58 -0
- package/dist/serviceTenants.js.map +1 -0
- package/dist/tools.d.ts +59 -0
- package/dist/tools.d.ts.map +1 -0
- package/dist/tools.js +60 -0
- package/dist/tools.js.map +1 -0
- package/dist/transport.d.ts +34 -0
- package/dist/transport.d.ts.map +1 -0
- package/dist/transport.js +227 -0
- package/dist/transport.js.map +1 -0
- package/dist/types.d.ts +399 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +2 -0
- package/dist/types.js.map +1 -0
- package/dist/webhooks.d.ts +38 -0
- package/dist/webhooks.d.ts.map +1 -0
- package/dist/webhooks.js +73 -0
- package/dist/webhooks.js.map +1 -0
- package/examples/letter-customer-research-demo.mjs +54 -0
- package/examples/letter-customer-research-store.d.mts +38 -0
- package/examples/letter-customer-research-store.mjs +186 -0
- package/examples/letter-customer-research.d.mts +39 -0
- package/examples/letter-customer-research.mjs +194 -0
- package/examples/letter-event-receiver.d.mts +10 -0
- package/examples/letter-event-receiver.mjs +55 -0
- package/examples/letter-event-store.d.mts +35 -0
- package/examples/letter-event-store.mjs +84 -0
- package/examples/letter-seer.d.mts +31 -0
- package/examples/letter-seer.mjs +78 -0
- package/examples/letter-service-tenants.d.mts +12 -0
- package/examples/letter-service-tenants.mjs +49 -0
- package/examples/research-webhook-receiver.mjs +29 -0
- package/package.json +75 -4
package/README.md
CHANGED
|
@@ -1,3 +1,599 @@
|
|
|
1
|
-
#
|
|
1
|
+
# OpenCrawl SDK
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
Watch competitor pages and research markets for each of your customers, from your server or from your LLM agents. ESM, Node.js 22+, no runtime dependencies. The SDK does not read environment variables or files: your server passes the API key.
|
|
4
|
+
|
|
5
|
+
```sh
|
|
6
|
+
npm install @getopencrawl/sdk
|
|
7
|
+
```
|
|
8
|
+
|
|
9
|
+
## Quickstart: signals and competitive research for your customers
|
|
10
|
+
|
|
11
|
+
```ts
|
|
12
|
+
import { OpenCrawl, agentTools } from '@getopencrawl/sdk';
|
|
13
|
+
|
|
14
|
+
const opencrawl = new OpenCrawl({ apiKey: process.env.OPENCRAWL_API_KEY! }); // your workspace admin key
|
|
15
|
+
const customers = opencrawl.customers({ keys }); // keys: see "Customer keys"
|
|
16
|
+
|
|
17
|
+
// Each customer gets an isolated OpenCrawl workspace, created on first use.
|
|
18
|
+
const acme = await customers.get({ id: 'acme', name: 'Acme', website: 'https://acme.com' });
|
|
19
|
+
|
|
20
|
+
// Watch competitor pages (Seer-style listeners). Same input again returns the same watch.
|
|
21
|
+
await acme.watch({ name: 'Greptile changelog', urls: ['https://www.greptile.com/changelog'], every: 'hour' });
|
|
22
|
+
|
|
23
|
+
// What changed since last time: real edits only (date, clock and "updated N days ago" noise is filtered).
|
|
24
|
+
const { signals, cursor } = await acme.signals({ since: savedCursor });
|
|
25
|
+
for (const s of signals) console.log(s.headline, s.changes); // save `cursor` for next time
|
|
26
|
+
|
|
27
|
+
// Competitive research with cited findings. Naming changelog pages gives the most reliable results.
|
|
28
|
+
const answer = await acme.ask('What did Greptile and Graphite ship recently, with dates?', {
|
|
29
|
+
urls: ['https://www.greptile.com/changelog', 'https://graphite.com/docs/cli-changelog'],
|
|
30
|
+
});
|
|
31
|
+
for (const f of answer.findings) console.log(f.claim, f.qualifiers, f.sources.map(s => s.url));
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
### Give your agents the tools
|
|
35
|
+
|
|
36
|
+
```ts
|
|
37
|
+
const tools = agentTools(acme);
|
|
38
|
+
|
|
39
|
+
// Anthropic
|
|
40
|
+
const msg = await anthropic.messages.create({ model, max_tokens: 2048, tools: tools.anthropic(), messages });
|
|
41
|
+
for (const block of msg.content)
|
|
42
|
+
if (block.type === 'tool_use') results.push({ type: 'tool_result', tool_use_id: block.id,
|
|
43
|
+
content: JSON.stringify(await tools.run(block.name, block.input)) });
|
|
44
|
+
|
|
45
|
+
// OpenAI: tools.openai() (Chat Completions) or tools.openaiResponses(); run calls with tools.run(name, args)
|
|
46
|
+
// Vercel AI SDK: import { jsonSchema } from 'ai'; generateText({ tools: tools.vercel(jsonSchema), ... })
|
|
47
|
+
// MCP clients: acme.mcp() returns { url, headers } for OpenCrawl's hosted MCP server, scoped to this customer
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
| Tool | What the agent can do |
|
|
51
|
+
| --- | --- |
|
|
52
|
+
| `opencrawl_watch_pages` | Watch pages hourly, daily or weekly |
|
|
53
|
+
| `opencrawl_list_watches` / `opencrawl_stop_watch` | Review or stop watches (history is kept) |
|
|
54
|
+
| `opencrawl_get_signals` | Read new changes since a cursor, with removed/added lines and a headline |
|
|
55
|
+
| `opencrawl_research` / `opencrawl_get_research` | Ask a question and get cited findings; read a long-running answer later |
|
|
56
|
+
|
|
57
|
+
Tool failures come back as `{ error: { type, message } }` so the agent can correct itself. `agentTools(acme, { prefix: 'oc_' })` renames tools.
|
|
58
|
+
|
|
59
|
+
### Customer keys
|
|
60
|
+
|
|
61
|
+
Each customer has its own key. `customers({ keys })` takes any store with `get(customerId)` and `set(customerId, key)` (database row, Redis, secrets manager). If a key is missing or rejected, the SDK obtains a new one with your admin key and saves it, so a lost key never strands a customer. Use one shared store when several processes serve customers; `memoryKeyStore()` (the default) suits a single process. Keep the admin key that created customers active: replacing it suspends them.
|
|
62
|
+
|
|
63
|
+
### Push instead of poll
|
|
64
|
+
|
|
65
|
+
`acme.client.webhooks` creates signed webhooks per watch (`research.changed` carries `substance`; date- and clock-only edits are never pushed). Verify deliveries with `verifyResearchWebhook`. Research answers have no webhook: poll `acme.answer(answerId)`.
|
|
66
|
+
|
|
67
|
+
### Limits
|
|
68
|
+
|
|
69
|
+
- `ask` reads at most three named pages; without names it searches the web and can miss a competitor. Asking the same question again within six hours returns the same answer (`reuseWithinMs` changes this); failed answers are retried.
|
|
70
|
+
- Long pages (over 128 KiB of text, or HTML up to 4 MiB) are read up to their first 128 KiB of text, which holds the newest entries on most changelogs; changes further down are not seen. Larger HTML is reported as unavailable.
|
|
71
|
+
- OpenCrawl generates a headline and a relevance rating (`high`, `medium`, `low`, with a reason) for most content changes, judged against the watch's `brief` (by default your customer's name, website and context). Ratings are model output: keep the changed lines as the evidence and apply your own positioning rule. Without a generated headline, one is built from the changed lines.
|
|
72
|
+
- Default limit: 100 customers per admin workspace; ask OpenCrawl to raise it.
|
|
73
|
+
|
|
74
|
+
## Lower-level API
|
|
75
|
+
|
|
76
|
+
Everything above is built on the typed client below, which remains available as `acme.client` or `new OpenCrawl({ apiKey })`.
|
|
77
|
+
|
|
78
|
+
## Read a finding you already reviewed
|
|
79
|
+
|
|
80
|
+
In Research, enter one customer brief, review the findings and supporting quotes, then open **Use this run in your app**. Its copied request contains the exact saved IDs. The equivalent server-side read is:
|
|
81
|
+
|
|
82
|
+
```ts
|
|
83
|
+
import { OpenCrawl } from '@getopencrawl/sdk';
|
|
84
|
+
|
|
85
|
+
const { OPENCRAWL_API_KEY: apiKey, OPENCRAWL_AGENT_ID: agentId, OPENCRAWL_RUN_ID: runId } = process.env;
|
|
86
|
+
if (!apiKey || !agentId || !runId) throw new Error('Set the server secret and exact saved agent and run IDs.');
|
|
87
|
+
const client = new OpenCrawl({
|
|
88
|
+
baseUrl: 'https://app.getopencrawl.com',
|
|
89
|
+
apiKey,
|
|
90
|
+
maxRetries: 0,
|
|
91
|
+
});
|
|
92
|
+
const run = await client.agents.runs.get(agentId, runId);
|
|
93
|
+
// Inspect status/outcome/gaps; keep claim, relevance, qualifiers and citations together.
|
|
94
|
+
// This reads saved research without starting another run.
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
Use a key for the same account and workspace that own the saved run. A customer's name in a brief is context, not an access boundary. For separate downstream customer access, create and read the agent using that customer's dedicated child key; substituting it into a human-owned run's request does not transfer that run. Keep every key on your server. See [API-first brief creation and recovery](#hosted-research-agents).
|
|
98
|
+
|
|
99
|
+
## Collect and search specific pages
|
|
100
|
+
|
|
101
|
+
Collections are a separate option when you need a reviewed source scope, an index or recurring page observations. This one-page manual check creates a collection:
|
|
102
|
+
|
|
103
|
+
```ts
|
|
104
|
+
import { OpenCrawl } from '@getopencrawl/sdk';
|
|
105
|
+
|
|
106
|
+
const client = new OpenCrawl({
|
|
107
|
+
baseUrl: 'https://app.getopencrawl.com',
|
|
108
|
+
apiKey: process.env.OPENCRAWL_API_KEY!, // your server's secret store
|
|
109
|
+
timeoutMs: 30_000,
|
|
110
|
+
maxRetries: 0, // explicit recovery is simplest for the first integration
|
|
111
|
+
// fetch: yourInstrumentedFetch, // optional, trusted transport injection
|
|
112
|
+
});
|
|
113
|
+
|
|
114
|
+
// Persist this key and the exact request in your own durable run record BEFORE sending.
|
|
115
|
+
const receipt = await client.collections.create({
|
|
116
|
+
name: 'Letter homepage pilot',
|
|
117
|
+
brief: 'Read the public product description with source citations.',
|
|
118
|
+
seed_urls: ['https://letter.dev/'],
|
|
119
|
+
allowed_origins: ['https://letter.dev'],
|
|
120
|
+
sitemap_urls: [],
|
|
121
|
+
page_limit: 1,
|
|
122
|
+
schedule_minutes: 0, // manual; no recurring crawl yet
|
|
123
|
+
include_media: false,
|
|
124
|
+
}, { idempotencyKey: durableRun.idempotencyKey });
|
|
125
|
+
|
|
126
|
+
// Persist the accepted receipt before starting a wait. Recover with these identifiers.
|
|
127
|
+
const job = await client.jobs.wait(receipt.job.id, { maxWaitMs: 120_000 });
|
|
128
|
+
// Keep job.status, job.progress and job.coverage visible in your product.
|
|
129
|
+
// A one-page bound may finish as "partial" while still returning useful evidence.
|
|
130
|
+
if (job.progress.indexed > 0) {
|
|
131
|
+
const evidence = await client.search({
|
|
132
|
+
query: 'What does Letter describe?',
|
|
133
|
+
collection_ids: [receipt.collection.id],
|
|
134
|
+
freshness: 'indexed',
|
|
135
|
+
limit: 10,
|
|
136
|
+
budget: 4096,
|
|
137
|
+
});
|
|
138
|
+
// Render passages, citations and dates alongside job.status and coverage.
|
|
139
|
+
// Failed/partial runs can retain evidence; that is not a completed collection.
|
|
140
|
+
// evidence.context is source material, not a generated answer.
|
|
141
|
+
} else {
|
|
142
|
+
// Show job.status, job.error and coverage. No indexed page is ready from this run.
|
|
143
|
+
// Retain the accepted IDs; do not create a replacement collection automatically.
|
|
144
|
+
}
|
|
145
|
+
```
|
|
146
|
+
|
|
147
|
+
`durableRun` in the example is your application-owned record, not an SDK global. Reuse its original idempotency key and exact descriptor to reconcile a lost create/refresh response. Do not generate a replacement key because a connection timed out. A changed descriptor with the same key is a conflict. The server decides whether a receipt is replayed.
|
|
148
|
+
|
|
149
|
+
`jobs.wait` can time out while the accepted job continues. Catch `OpenCrawlTimeoutError`, retain the job ID, and read or wait on that same job later. An empty result means there is no matching indexed evidence for that request; it is not an answer about the whole company. The [actual cloud acceptance](../evals/deployment-verification/service-tenant-sdk-cloud-2026-10-24.json) verified one indexed Letter page and two citations through SDK and MCP, plus customer isolation and revocation. It is an integration check, not a retrieval-quality benchmark.
|
|
150
|
+
|
|
151
|
+
`jobs.wait` returns all terminal receipts, including `failed` and `cancelled`; those states do **not** throw an API error. Inspect both status and `progress.indexed`. Zero indexed pages is empty or failed work, while a partial run with indexed pages may already support a useful cited query.
|
|
152
|
+
|
|
153
|
+
## Hosted research agents
|
|
154
|
+
|
|
155
|
+
Use a customer and goal for a bounded external research run. Optional `collection_ids` add existing permitted evidence; `[]` means external research only. The server saves source-attributed findings, qualifications and citations. The `relevance` string remains for compatibility; new runs use a neutral label and historical values remain unchanged. It does not silently create a crawl or a recurring schedule.
|
|
156
|
+
|
|
157
|
+
```ts
|
|
158
|
+
// Save this exact brief and key in your own durable operation record before sending.
|
|
159
|
+
const accepted = await client.agents.create({
|
|
160
|
+
customer: {name: 'Example customer', website: 'https://example.org/', context: 'Finance workflow automation'},
|
|
161
|
+
goal: 'Find public workflow changes relevant to supplier invoice automation; preserve limitations.',
|
|
162
|
+
collection_ids: [],
|
|
163
|
+
schedule_minutes: 0,
|
|
164
|
+
}, {idempotencyKey: durableRun.idempotencyKey});
|
|
165
|
+
// Persist both IDs before waiting; waiting never starts another run.
|
|
166
|
+
const run = await client.agents.runs.wait(accepted.agent.id, accepted.run.id, {maxWaitMs: 300_000});
|
|
167
|
+
// Inspect run.status/outcome/gaps and keep claim, relevance, qualifiers and citations together.
|
|
168
|
+
const saved = await client.agents.runs.get(accepted.agent.id, accepted.run.id);
|
|
169
|
+
```
|
|
170
|
+
|
|
171
|
+
`agents.create` and `agents.runs.create` send once, even when transport read retries are configured. Recover uncertain admission with `agents.receipt('create', {idempotencyKey})` or `agents.receipt('run:AGENT_ID', {idempotencyKey})`; this returns the original historical receipt. Follow its run ID for current findings. `runs.wait` returns every terminal status honestly and a local timeout does not cancel server work. `agents.list/get/update` manage only the current identity's private agents. Pause cancels unfinished work; resume does not enqueue it.
|
|
172
|
+
|
|
173
|
+
Use a dedicated child key for each downstream customer. A customer name is context, never an access boundary. Citation offsets and `body_sha256` describe supplied evidence, possibly a snippet or passage; the SDK cannot verify the complete hash from the quote alone. `published_at` is a literal provider label. See the [full contract, recovery and resource limits](../docs/research-agents.md). Install a matching released server/client package; existing immutable package URLs do not change when source code is committed.
|
|
174
|
+
|
|
175
|
+
Fetched source pages use `source_kind: 'web_page'` and a separate `page` record: requested/final URLs, full extracted-page `body_sha256`, codepoint window `start`/`end`, `total_chars`, and extractor `public-static-v3`. The citation URL equals the final URL. Citation `body_sha256` still identifies the supplied window, and citation offsets remain window-relative; full-page quote positions are `page.start + citation.start/end`. These hashes are provenance metadata, not independently verified from the quote. Historical `web_search` and `indexed` citations remain readable and omit `page`.
|
|
176
|
+
|
|
177
|
+
Daily research is separate and opt-in. After the initial run finishes, read `client.agents.schedule.get(agentId)`, then call `client.agents.schedule.update(agentId, {expected_revision: schedule.revision, enabled: true})`. This uses the schedule's own revision, starts no immediate run, and sets the first due time 24 hours later. Use `enabled: false` to stop future admissions and pending daily work that has not crossed the dispatch boundary. Manual runs and already dispatched daily runs remain unchanged. After an uncertain write, read the schedule before another write; no automatic retries or replacement revisions are used. Inspect `blocked_reason` (explicit review/resume required) and `deferred_reason` (capacity wait for the same slot).
|
|
178
|
+
|
|
179
|
+
`client.agents.runs.repeats(agentId, runId)` reads separate `first_seen`/`exact_repeat` references. It never filters the saved run or claims semantic novelty. Historical/nonterminal runs may be `not_assessed`. Updated clients accept truthful `Agent.schedule_minutes: 0 | 1440`; older clients reject daily agents and mixed agent lists. Direct saved Run DTOs and historical manual receipts are unchanged. Create remains manual (`schedule_minutes: 0`); scheduling cannot be enabled accidentally through a brief. See the [daily scheduling and recovery contract](../docs/research-agents.md#opt-in-daily-research-and-exact-repeats).
|
|
180
|
+
|
|
181
|
+
### Give feedback on a saved finding
|
|
182
|
+
|
|
183
|
+
This addition requires the feedback-capable server and package; consult [current release status](../STATUS.md) before using it. Feedback is an explicit customer preference or correction. It does not edit the saved finding, verify its truth or start another run.
|
|
184
|
+
|
|
185
|
+
```ts
|
|
186
|
+
const feedback = client.agents.runs.feedback;
|
|
187
|
+
const current = await feedback.get(agentId, runId, findingId);
|
|
188
|
+
const savedFeedback = await feedback.set(agentId, runId, findingId, {
|
|
189
|
+
expected_revision: current.feedback.revision,
|
|
190
|
+
rating: 'useful', // useful | irrelevant | incorrect | outdated
|
|
191
|
+
note: 'Keep focusing on pricing changes that affect this customer.',
|
|
192
|
+
});
|
|
193
|
+
// An explicit removal uses the latest reviewed feedback revision.
|
|
194
|
+
await feedback.delete(agentId, runId, findingId, savedFeedback.feedback.revision);
|
|
195
|
+
```
|
|
196
|
+
|
|
197
|
+
Writes send once. On a conflict or uncertain response, read the same feedback, compare its saved value and review it before another write. Notes allow at most 1,000 Unicode characters and 4,096 UTF-8 bytes, without NUL or malformed Unicode. Removal clears the stored rating/note and retains a revision tombstone. The optional Node24 customer recipe exposes the same operations as `research.feedback`, `research.setFeedback` and `research.deleteFeedback`, using its existing customer authorization and saved-run binding.
|
|
198
|
+
|
|
199
|
+
Feedback is saved with its finding. Automatic use of history and feedback in subsequent research is experimental and disabled by default; saving feedback does not enable it. Its initial comparison did not establish the required selection benefit. Any operator-enabled experiment still requires current evidence for new claims and does not train a model. [Memory and feedback contract](../docs/research-agents.md#finding-feedback-and-bounded-memory).
|
|
200
|
+
|
|
201
|
+
### Poll saved daily findings from your server
|
|
202
|
+
|
|
203
|
+
There is no research-agent completion webhook; collection source-change webhooks are a different feature. Use `agents.runs.list(agentId, {limit, cursor})` to discover every retained admission, then read pending IDs directly. A page is admission order, **not a terminal-update feed**. `store` below is your application's durable adapter, not an SDK service. Its per-agent lease must serialize this entire function, and its transaction must commit run rows, pending membership and cursor together:
|
|
204
|
+
|
|
205
|
+
<!-- research-history-consumer:start -->
|
|
206
|
+
```js
|
|
207
|
+
async function pollResearch(client, agentId, store, signal) {
|
|
208
|
+
return store.withAgentLease(agentId, async () => {
|
|
209
|
+
const cursor = await store.getCursor(agentId); // undefined on first use
|
|
210
|
+
const page = await client.agents.runs.list(agentId, {limit: 25, cursor, signal});
|
|
211
|
+
const pending = run => ['queued', 'running'].includes(run.status);
|
|
212
|
+
await store.transaction(async tx => {
|
|
213
|
+
for (const run of page.items) {
|
|
214
|
+
await tx.upsertRun(run.id, run);
|
|
215
|
+
await tx.setRunPending(agentId, run.id, pending(run));
|
|
216
|
+
}
|
|
217
|
+
await tx.setCursor(agentId, page.next_cursor);
|
|
218
|
+
});
|
|
219
|
+
// Bound each invocation; remaining pending IDs stay durable for the next poll.
|
|
220
|
+
for (const runId of await store.listPending(agentId, {limit: 25})) {
|
|
221
|
+
const run = await client.agents.runs.get(agentId, runId, {signal});
|
|
222
|
+
await store.transaction(async tx => {
|
|
223
|
+
await tx.upsertRun(run.id, run);
|
|
224
|
+
await tx.setRunPending(agentId, run.id, pending(run));
|
|
225
|
+
});
|
|
226
|
+
}
|
|
227
|
+
return {hasMoreAdmissions: page.has_more};
|
|
228
|
+
});
|
|
229
|
+
}
|
|
230
|
+
```
|
|
231
|
+
<!-- research-history-consumer:end -->
|
|
232
|
+
|
|
233
|
+
Run one bounded poll at a time per agent, with a caller-owned abort deadline and backoff. `upsertRun` must be idempotent by run ID; commit each update and its pending flag atomically. A lost local commit acknowledgement may cause a safe reread, not duplicate delivery. If downstream actions need delivery guarantees, write a run-ID-keyed outbox in the same transaction and let your own sender deduplicate. This example persists results; it does not promise exactly-once effects in another system.
|
|
234
|
+
|
|
235
|
+
Keep polling the returned cursor even when `has_more` is false: later admissions can appear. Also retain and poll all pending IDs until terminal, even after their admission page is checkpointed. Terminal `partial` may have zero findings; `failed`, `unknown`, `needs_input`, and `cancelled` are stored outcomes, never permission to create a replacement run. Keep every qualifier/citation. Repeat metadata is a separate optional read through `runs.repeats` and does not remove original findings.
|
|
236
|
+
|
|
237
|
+
The envelope is `{items, next_cursor, has_more}` with a nonnull cursor, including an empty initial/tail page. Default limit 25, maximum 100, and a 1 MiB compact UTF-8 envelope cap can yield fewer items than requested. No SDK or CLI auto-pagination occurs. Cursors are opaque and bound to agent/owner/workspace; do not parse, synthesize or transplant them. Legacy backfill has deterministic iteration order, not reconstructed historical commit order. Current authorization is checked on every page and exact-run read; any inaccessible returned historical evidence denies the whole page. Preserve the old cursor on that error and surface the obstruction for operator resolution—never skip it silently. Pending-read failure retains that ID. A temporary read failure can be retried under your caller's bounded policy; it must never trigger an admission, resume or rerun. Existing saved Run and receipt shapes remain unchanged.
|
|
238
|
+
|
|
239
|
+
### Durable customer research recipe (Node 24)
|
|
240
|
+
|
|
241
|
+
`letter-customer-research.mjs` supplies the admission/recovery and polling flow above; `letter-customer-research-store.mjs` supplies its real SQLite store. These optional examples require **Node.js 24+**. The SDK's general Node.js 22+ floor is unchanged. Release `43d278d` serves the recipe; its [clean install, typed exports and offline demonstration](../evals/deployment-verification/operator-acquisition-cloud-2026-10-24.json) are verified. This does not establish an installed Letter integration. From the repository root, the offline demonstration also runs directly:
|
|
242
|
+
|
|
243
|
+
```sh
|
|
244
|
+
npm --prefix sdk ci
|
|
245
|
+
npm --prefix sdk run build
|
|
246
|
+
node sdk/examples/letter-customer-research-demo.mjs --mock
|
|
247
|
+
```
|
|
248
|
+
|
|
249
|
+
In your server project's directory, install the current immutable SDK using the command supplied by the app or package manifest above, then run the bundled demonstration. Retain the resolved URL and integrity in your lockfile:
|
|
250
|
+
|
|
251
|
+
```sh
|
|
252
|
+
node node_modules/@getopencrawl/sdk/examples/letter-customer-research-demo.mjs --mock
|
|
253
|
+
```
|
|
254
|
+
|
|
255
|
+
It exercises existing tenant provisioning, one research admission, saved-run reads and a history checkpoint with fake transport and real temporary SQLite. It uses synthetic credentials and prints only outcome counts. No hosted calls occur. The demo's in-memory vault is a mock; it is not a production secret-store adapter.
|
|
256
|
+
|
|
257
|
+
For your server, save this wiring as `letter-research.mjs`. Your host supplies all three arguments; no credentials are read from request parameters or environment variables by this function:
|
|
258
|
+
|
|
259
|
+
```js
|
|
260
|
+
import { createLetterCustomerResearch } from '@getopencrawl/sdk/examples/letter-customer-research.mjs';
|
|
261
|
+
import { LetterCustomerResearchStore } from '@getopencrawl/sdk/examples/letter-customer-research-store.mjs';
|
|
262
|
+
|
|
263
|
+
export function openCustomerResearch({ filename, authorize, encryptedVault }) {
|
|
264
|
+
// Absolute path under an existing private directory owned by this process, mode 0700.
|
|
265
|
+
const store = new LetterCustomerResearchStore(filename);
|
|
266
|
+
const research = createLetterCustomerResearch({
|
|
267
|
+
store,
|
|
268
|
+
baseUrl: 'https://app.getopencrawl.com',
|
|
269
|
+
authorize, // async (trustedRequestContext, customerId) => boolean
|
|
270
|
+
secrets: { get: secretRef => encryptedVault.get(secretRef) },
|
|
271
|
+
});
|
|
272
|
+
return { store, research }; // Close store during your server's orderly shutdown.
|
|
273
|
+
}
|
|
274
|
+
```
|
|
275
|
+
|
|
276
|
+
`authorize` must check Letter's authenticated principal and current customer membership, not merely compare a browser-supplied customer ID. `encryptedVault.get` resolves an opaque server-owned reference to that customer's child key, never the parent key. The host owns encryption, access control, rotation and revocation. The recipe checks host authorization and the mapping generation around asynchronous work, but cannot validate the host's authorization policy or vault implementation.
|
|
277
|
+
|
|
278
|
+
During trusted setup, use the existing [service-tenant helpers](#embedded-customer-workspaces), durably save the first-response child key in your encrypted vault, then call `store.bindCustomer({customerId, tenantId, secretRef, active: true})` with that saved mapping. Do this after confirmed provisioning or explicit recovery, not on every request. Tenant and secret references cannot be shared across customers; a customer's tenant cannot be reassigned. Rebind on key rotation even if the reference stays the same, or set `active: false` to stop local use. This fences in-flight local results; it does not revoke the hosted credential. Keep the original provisioning parent key active. No provisioning, replacement tenant, key rotation or key recovery is automatic.
|
|
279
|
+
|
|
280
|
+
Route explicit host actions to these methods with trusted `context`, the authorized `customerId`, and a caller-owned abort signal:
|
|
281
|
+
|
|
282
|
+
| Action | Call | What is durable |
|
|
283
|
+
| --- | --- | --- |
|
|
284
|
+
| First reviewed brief | `research.start(context, customerId, brief, {signal})` | Normalized brief and random request key are committed before one POST; accepted agent/run IDs are saved. |
|
|
285
|
+
| Uncertain first response | `research.recover(context, customerId, {signal})` | GET-only recovery with the original key; missing or denied receipts leave that intent intact. |
|
|
286
|
+
| Deliberately repeat the existing brief | `research.rerun(context, customerId, previousRunId, {signal})` | Freshly checks the known latest terminal run, then saves its ID, current revision and a random request key before one POST. |
|
|
287
|
+
| Reconcile a repeat response | `research.recoverRerun(context, customerId, previousRunId, {signal})` | GET-only recovery using that repeat's saved key; can explicitly retire a confirmed refusal, never creates a replacement run. |
|
|
288
|
+
| Display saved evidence | `research.read(context, customerId, runId, {signal})` | Exact known run is read with current access; failure does not return cached evidence. |
|
|
289
|
+
| Inspect saved source context | `research.source(context, customerId, runId, citation.id, {signal})` | One retained source window for a known run is read with current access and verified text hash; source text is not cached in the recipe store. |
|
|
290
|
+
| Continue history | `research.poll(context, customerId, {signal})` | One page of up to 25 admissions commits runs, pending IDs and cursor atomically; up to 25 pending IDs are then checked. |
|
|
291
|
+
|
|
292
|
+
`brief` is the normal manual `ResearchAgentInput`. There is one initial create intent per customer mapping. After a timeout or lost response, use `recover`, never another `start`; even a crash between committing intent and sending the POST requires explicit review if no receipt exists. A local timeout does not cancel server work. Each call defaults to 30 seconds and permits `timeoutMs` up to 60 seconds. No method automatically retries a mutation, pages history, runs a timer or enables daily research. After reviewing the initial result, call `research.schedule(context, customerId)` and then `research.setSchedule(context, customerId, {expected_revision: schedule.revision, enabled: true})` explicitly. These methods use the saved customer mapping and current authorization. A write starts no immediate run; reconcile a conflict or uncertain outcome with a separate `schedule` read.
|
|
293
|
+
|
|
294
|
+
For **Run again**, pass the saved run the user reviewed as `previousRunId`. It must be known to this customer mapping and freshly observed as the agent's latest terminal run, including partial or failed outcomes. That ID identifies one deliberate repeat: duplicate clicks, another process and restarts cannot send its POST again. The next deliberate repeat uses the new terminal run's ID. `rerun` does not change the brief or select an experimental policy; omitted policy uses the server's current default. The accepted IDs are immediately available to the existing `read`, `source` and `poll` methods.
|
|
295
|
+
|
|
296
|
+
The latest-run check is a read before dispatch, **not an atomic latest-predecessor condition**: an external run can finish between that read and the POST. The backend still enforces the captured agent revision and its one-unfinished-run rule; the recipe guarantees at most one local POST for each saved intent. An uncertain or refused admission keeps its key and blocks further recipe reruns until explicitly reconciled. Use `recoverRerun` with the same predecessor, never another key or `start`. After a restart, trusted server code can find that predecessor in `store.pendingRerun(customerId)` and then call the authorized recovery method.
|
|
297
|
+
|
|
298
|
+
If the original POST returned **409 or 429** while customer access and the mapping were still valid, the store records that refusal. An explicit `recoverRerun` can retire it only after the original-key receipt returns 404 and a fresh authorized read confirms the same agent and brief under the same mapping generation. It archives the original key and returns `{refused: true, previousRunId}`. It sends no POST. After addressing the conflict or limit, a separate user action may call `rerun` again with that predecessor; it checks the latest run again and saves a new key. A receipt found by recovery is always reconciled as an admission instead.
|
|
299
|
+
|
|
300
|
+
```js
|
|
301
|
+
const recovered = await research.recoverRerun(context, customerId, previousRunId);
|
|
302
|
+
if ('refused' in recovered) {
|
|
303
|
+
// Tell the user this attempt was refused. Offer Run again once the cause is resolved.
|
|
304
|
+
// Only that later deliberate action calls research.rerun(..., recovered.previousRunId).
|
|
305
|
+
} else {
|
|
306
|
+
const run = await research.read(context, customerId, recovered.runId);
|
|
307
|
+
// Display this same admitted run using the existing result renderer.
|
|
308
|
+
}
|
|
309
|
+
```
|
|
310
|
+
|
|
311
|
+
A recovery 404 **alone never clears an intent**. Timeouts, connection loss, 5xx responses, malformed receipts, a crash before dispatch, old intents without a saved refusal, and changed customer mappings remain unresolved for operator review. Accepted receipt identities are never reset. Recovery does not automatically resubmit or create another initial brief.
|
|
312
|
+
|
|
313
|
+
Known HTTP refusals expose `CustomerResearchRemoteError` with a finite local `code` and HTTP `status`, without remote response text. For example, `access_denied`, `rate_limited`, `conflict` and `source_not_retained` let Letter explain the obstruction. Uncertain outcomes retain their existing recovery path. A refusal never authorizes another `start`; only the explicit rerun reconciliation above can retire a witnessed repeat refusal.
|
|
314
|
+
|
|
315
|
+
Poll again under your host's bounded scheduler even at an empty tail; `pending` is the total unfinished count, while each invocation checks at most 25 IDs. `observedRunIds` identifies the rows read in that invocation. After restart, `store.storedRuns(customerId, {offset, limit: 25})` enumerates only local IDs/status; use `research.read` before displaying a selected result. These local metadata methods are trusted server operations, not a public authorization boundary. Partial, empty, failed and unknown outcomes stay stored; they do not trigger replacement research. This recipe does not provide an external-effect outbox or an agent-completion webhook.
|
|
316
|
+
|
|
317
|
+
Use `source` after selecting a citation from that run. It applies the same customer authorization, mapping, vault and deadline checks as `read`, including a fresh access check around the request. The returned `source.text` is the admitted window, possibly part of a page; quote positions are Unicode codepoints within that text. Missing historical context, revoked access or invalid source data rejects the read, without returning previous text or fetching the current website. A successful source read confirms retained context, not complete coverage or usefulness.
|
|
318
|
+
|
|
319
|
+
The database stores private briefs, receipts and full saved findings in plaintext, **not credentials**. Protect its directory, backups and retention accordingly; do not expose its low-level methods or log records. Defaults admit 1,000 run rows and 32 MiB of serialized run data, plus at most 1,000 compact active rerun intents and archived refusals combined (the same `maxRuns` bound). Rerun receipts retain a digest and accepted IDs; no extra copy of the full receipt is stored. Refusal archives preserve the old key, revision and finite refusal status. SQLite overhead and other tables are additional. Capacity errors preserve the checkpoint and require operator action. The [six-case development baseline](../evals/results/operator-development-six-case-4000a80-2026-10-24.json) did not clear complete requested coverage: recovery and isolation tests are not a research-quality guarantee. Keep source claims, qualifiers, citations, status and gaps together and review them before downstream action.
|
|
320
|
+
|
|
321
|
+
## API
|
|
322
|
+
|
|
323
|
+
- `agents.runs.source(agentId, runId, citation.id, {signal?, timeoutMs?})` → one saved admitted source window, with exact text and a locally verified SHA256. It checks current access and never crawls a URL. Source offsets count Unicode codepoints; use `Array.from(source.text).slice(citation.start, citation.end).join('')`. A window may cover only part of a page. Older runs without snapshots return `source_not_retained`; do not replace missing historical context with current website content.
|
|
324
|
+
- `collections.create(input, {idempotencyKey, signal?, timeoutMs?})` → collection/job receipt.
|
|
325
|
+
- `collections.list({limit?, cursor?, signal?})` / `get(id)` → current permitted descriptors.
|
|
326
|
+
- `collections.configure(id, {expected_revision, page_limit, schedule_minutes, include_media}, {idempotencyKey})` → accepted settings and next scheduled check; read `collections.get(id)` for the current state after recovery.
|
|
327
|
+
- `collections.refresh(id, {expected_revision?}, {idempotencyKey, signal?})` → job receipt. A revision conflict requires rereading and reviewing the descriptor.
|
|
328
|
+
- `collections.events(id, {limit?, cursor?})` → one page of observation events.
|
|
329
|
+
- `collections.event(collectionId, eventId, {budget?, beforeStart?, afterStart?, signal?, timeoutMs?})` → one bounded read of the event's retained before/after source text.
|
|
330
|
+
- `collections.iterate(options)` / `iterateEvents(id, options)` → bounded async iterators; `maxPages` defaults to 1000. For checkpointed processing, consume explicit pages and persist `next_cursor` only after successful handling.
|
|
331
|
+
- `jobs.get(id)` / `jobs.wait(id, {maxWaitMs?, pollIntervalMs?, onProgress?, signal?})` → actual job state. Waiting does not create, restart, or cancel a server job.
|
|
332
|
+
- `search({query, collection_ids?, freshness?, max_age_seconds?, limit?, budget?})` → current evidence plus freshness/refresh receipts.
|
|
333
|
+
- `context({query, source_ids?, limit?, budget?})` → existing source-scoped context API.
|
|
334
|
+
|
|
335
|
+
An omitted collection/source filter means all accessible content. An explicit empty research collection array is rejected by the server; the SDK never broadens it. Explicit empty legacy source selectors stay empty. Query text stays in the POST body. Pagination cursors are opaque values, never URLs to fetch.
|
|
336
|
+
|
|
337
|
+
`freshness: 'refresh'` can enqueue background work. Its response reports `refresh_jobs`; it does not promise the response includes newly fetched pages. Wait on those jobs, then make an indexed search if your workflow requires the fresh result. Inspect per-job coverage and result dates. Indexed dates and observation times are not source-authored change times.
|
|
338
|
+
|
|
339
|
+
## Read a selected event's source evidence
|
|
340
|
+
|
|
341
|
+
Use the event ID returned by `collections.events`, `readSeerRun`, or a verified research webhook. Read it with the key and collection belonging to the same customer; never substitute the parent administrator's key for a downstream customer's key.
|
|
342
|
+
|
|
343
|
+
```ts
|
|
344
|
+
const event = await client.collections.event(collectionId, eventId, {budget: 16000});
|
|
345
|
+
// Review event.before?.text and event.after?.text together with kind, gap and dates.
|
|
346
|
+
// A caller can explicitly request more of one immutable observation:
|
|
347
|
+
if (event.after?.next_start_char !== null && event.after?.next_start_char !== undefined) {
|
|
348
|
+
const continued = await client.collections.event(collectionId, eventId, {
|
|
349
|
+
budget: 16000,
|
|
350
|
+
...(event.before ? {beforeStart: event.before.start_char} : {}),
|
|
351
|
+
afterStart: event.after.next_start_char,
|
|
352
|
+
});
|
|
353
|
+
// Before replacing displayed evidence, require the same before/after version IDs
|
|
354
|
+
// and hashes. Keep each returned window's own offsets; this is not a merged diff.
|
|
355
|
+
}
|
|
356
|
+
```
|
|
357
|
+
|
|
358
|
+
The default budget is 16,000 UTF-8 **source-text** bytes shared between both sides (range 256–32,000); bounded JSON metadata is additional. `start_char`, `end_char`, `full_length` and continuation offsets count Unicode codepoints, not JavaScript UTF-16 units or bytes. `text` is the exact retained slice `[start_char, end_char)`. An explicit offset can request earlier unchanged context; an offset at the body's end returns an empty window, while one past the end or for a null side fails. No continuation is fetched automatically.
|
|
359
|
+
|
|
360
|
+
Default windows enclose the changed region when both bodies differ. This region can include unchanged or moved text and is not a minimal diff. `equal_body` retains title-only changes; `single_observation` and `no_observation` preserve missing sides without inventing content. A continuation can include source text beyond the original changed region. Baselines are observations, unavailable checks do not prove removal, and `gap` means the exact change time is unknown. None of these labels establishes purchase intent or a verified vendor capability.
|
|
361
|
+
|
|
362
|
+
`version_sha256` identifies the full UTF-8 `title + NUL + body`; `body_sha256` identifies only the full body. The SDK validates their format and the window's byte/offset metadata; it cannot verify a full-body hash from a partial window. Every read checks current authorization and the exact retained event/version lineage. Missing or no-longer-accessible history returns an error, with no fallback to a current document or live URL. The read starts no crawl, search, embedding or answer generation; ordinary account authorization can still contact WorkOS. These client methods require a server release that includes the event reader; installing a new client alone does not add the endpoint.
|
|
363
|
+
|
|
364
|
+
## Failures, cancellation, and transport safety
|
|
365
|
+
|
|
366
|
+
The origin must be HTTPS (HTTP is allowed only on loopback for local tests), without a path, embedded credentials, query, or fragment. Redirects are never followed. The built-in fetch omits cookies and sends the API key only to this origin. Your injected transport is trusted application code and must honor the same redirect/credential boundary. Browser execution is rejected; never bundle an API key into frontend code.
|
|
367
|
+
|
|
368
|
+
Timeout covers the complete request, response body, retries, and backoff. An `AbortSignal` cancels the client operation; it does not undo an accepted server mutation. Persist receipts promptly. If acceptance is uncertain, reconcile with the original idempotency key and exact input. After reload/restart, use your saved receipt/job ID; the SDK has no hidden filesystem persistence.
|
|
369
|
+
|
|
370
|
+
Read-only requests and ordinary idempotency-keyed mutations may retry network failures or HTTP 429/502/503/504, at most `maxRetries` (default 2). Backoff is bounded; a `Retry-After` longer than 10 seconds is surfaced without an automatic retry. Authentication/permission/conflict/validation failures are never retried. Revision-sensitive updates/deletes, refreshing search, and service-tenant create/rotate/revoke are not automatically retried. No returned evidence is cached by the SDK.
|
|
371
|
+
|
|
372
|
+
Errors extend `OpenCrawlError`: `OpenCrawlAPIError` (`status`, optional `code`, `requestId`, `retryAfterMs`), `OpenCrawlConnectionError` (`acceptanceUnknown`, original `idempotencyKey`), `OpenCrawlTimeoutError`, `OpenCrawlAbortError`, `OpenCrawlValidationError`, and `OpenCrawlProtocolError`. Do not treat an error as proof a write failed before acceptance. API messages are bounded; headers/bodies/transport causes are not attached. Avoid logging queries, evidence, and raw application secrets in your own instrumentation.
|
|
373
|
+
|
|
374
|
+
## Letter Seer example
|
|
375
|
+
|
|
376
|
+
`examples/letter-seer.mjs` exports `prepareSeer`, `readSeerRun` and `packSeerPassages` for Letter's server-side runner. Defaults are one page, manual refresh, and no media. Pass larger bounds, cadence, or media explicitly after reviewing the pilot. This is an adapter example, **not a claim that a Letter API integration has already been installed**. It returns receipts, actual per-run coverage, source-linked events, and current scoped evidence. It never sends email, creates Letter resources, or emits an external webhook.
|
|
377
|
+
|
|
378
|
+
These explicit package imports work after installing the hosted tarball, with TypeScript declarations included:
|
|
379
|
+
|
|
380
|
+
```ts
|
|
381
|
+
import { prepareSeer, readSeerRun, packSeerPassages } from '@getopencrawl/sdk/examples/letter-seer.mjs';
|
|
382
|
+
import { provisionCustomer, clientForCustomer } from '@getopencrawl/sdk/examples/letter-service-tenants.mjs';
|
|
383
|
+
```
|
|
384
|
+
|
|
385
|
+
`readSeerRun` omits changes whose only differences are dates, clock times, relative ages or whitespace (pass `includeVolatile: true` to keep them) and gives each event its removed/added lines as `changes` and its `substance`, for the host's positioning rule. Older events have `changes: null`. `readSeerRun` returns `processing` while the job is active. A terminal run with zero indexed pages returns `empty` or `needs_attention`, with its job receipt and no invented evidence. Indexed evidence from a failed/partial run retains `needs_attention`; callers must keep that status visible. A `review` state still requires checking the actual returned citations—it is not a verified answer.
|
|
386
|
+
|
|
387
|
+
For a server-side agent that needs more supporting detail, `packSeerPassages(run.evidence)` packages the **already returned** `results[].content` passages. It makes no request and leaves the search response and compact context unchanged. The fixed 32,768-byte cap includes the complete compact-JSON UTF-8 envelope, source text, provenance and omission fields. Whole passages are admitted in result order; an oversized passage is omitted intact, and later fitting passages may still be included. Inspect `all_returned_passages_included` and `omitted_citations`; no passage is shortened or citation renumbered.
|
|
388
|
+
|
|
389
|
+
The pack preserves original document/chunk identities, codepoint source ranges, URLs, titles, observation/version metadata, dates, freshness and gaps. Its ranges belong to full returned passages, so do not reuse `evidence.citations[].context_span`, which refers to the separate compact context. A complete pack means all **returned passages** fit; it does not mean all page text, relevant facts or qualifications were retrieved. Missing retrieval remains missing. The host must treat every source string as untrusted data, keep it separate from instructions, and authorize retained copies; packing does not renew platform access or verify a vendor claim. The helper is an explicit option for the existing example, and does not change `readSeerRun` behavior.
|
|
390
|
+
|
|
391
|
+
From the SDK source checkout (the `scripts` directory is not included in the installed package):
|
|
392
|
+
|
|
393
|
+
```sh
|
|
394
|
+
node scripts/demo.mjs --mock
|
|
395
|
+
# Live against the canonical application, using approved public URLs:
|
|
396
|
+
# Set OPENCRAWL_BASE_URL, OPENCRAWL_API_KEY in the server secret environment.
|
|
397
|
+
# Set RESEARCH_IDEMPOTENCY_KEY to your saved durable key, RESEARCH_SEED_URLS
|
|
398
|
+
# to a JSON URL array; or resume RESEARCH_COLLECTION_ID + RESEARCH_JOB_ID.
|
|
399
|
+
node scripts/demo.mjs
|
|
400
|
+
```
|
|
401
|
+
|
|
402
|
+
A baseline is a first observation, not a newly occurring business event. Unavailable pages do not prove removed jobs, closed companies, or missing capabilities. Event gaps and truncation must remain visible. Apply positioning/intent interpretation in the host product with cited evidence, separately from these observations. Deduplicate by event ID and acknowledge a cursor only after the host has processed its page.
|
|
403
|
+
|
|
404
|
+
Collection creation supports up to 50 seed URLs, 20 explicitly approved origins, and 5,000 pages per run. The optional brief describes research intent; it does not broaden crawl scope. For an ongoing collection, choose `schedule_minutes: 60` (hourly) or `1440` (daily) at creation. Keep the successful pilot and use `collections.configure` to review and change its page limit, cadence and media choice. Source selection stays fixed. Saving schedules future work without starting an immediate crawl; active or unfinished work must finish first. See [collection settings](../docs/research-configuration.md) for revision checks and recovery with the original operation key.
|
|
405
|
+
|
|
406
|
+
To pause, call `collections.update(id, {status: "paused", expected_revision: collection.revision})`; resume uses `status: "active"` with the current revision. Updates are never automatically retried; reload the descriptor after an uncertain response or revision conflict.
|
|
407
|
+
|
|
408
|
+
## Signed change notifications
|
|
409
|
+
|
|
410
|
+
Webhook subscriptions are collection-owner operations using a full API key. An endpoint must be a public HTTPS URL without credentials, query parameters or a fragment. The server validates destinations and does not follow redirects. The SDK only sends registration requests to OpenCrawl; it never sends your API key to the receiver.
|
|
411
|
+
|
|
412
|
+
```ts
|
|
413
|
+
const subscription = await client.webhooks.create(collection.id, {
|
|
414
|
+
endpoint: 'https://your-server.example/hooks/opencrawl',
|
|
415
|
+
expected_collection_revision: collection.revision,
|
|
416
|
+
event_types: ['changed', 'recovered'], // also the default
|
|
417
|
+
}, {idempotencyKey: durableSubscription.createKey});
|
|
418
|
+
|
|
419
|
+
// Save this first-response-only secret in your server secret store, never in a browser.
|
|
420
|
+
if (subscription.secret !== null) await secretStore.save(subscription.webhook.id, subscription.secret);
|
|
421
|
+
```
|
|
422
|
+
|
|
423
|
+
Create and rotate return `{webhook, secret, replayed}`. After a committed response is lost, repeating the **same descriptor and key** returns `secret: null`; it does not recover the secret or rotate automatically. If the secret was not saved, read the current subscription revision and explicitly rotate with a new durable operation key. Rotation changes the revision and invalidates pending deliveries from the previous configuration.
|
|
424
|
+
|
|
425
|
+
- `webhooks.list(collectionId)` / `get(webhookId)` → subscriptions with endpoint origin only.
|
|
426
|
+
- `webhooks.update(id, {expected_revision, status: 'active' | 'paused'})` → current subscription.
|
|
427
|
+
- `webhooks.remove(id, {expected_revision})` → revoked subscription. Neither update nor revoke is retried automatically.
|
|
428
|
+
- `webhooks.rotate(id, {expected_revision}, {idempotencyKey})` → first-response secret receipt.
|
|
429
|
+
- `webhooks.test(id, {expected_revision}, {idempotencyKey})` → queued delivery receipt; queueing is not confirmed delivery.
|
|
430
|
+
- `webhooks.deliveries(id, {limit?, offset?})` → bounded page with real delivery state, attempt count, webhook/collection revisions and `may_have_been_delivered`.
|
|
431
|
+
- `webhooks.replay(id, deliveryId, {expected_revision, expected_replay_count}, {idempotencyKey})` → explicit replay receipt. The server rejects an obsolete configuration or ineligible delivery.
|
|
432
|
+
|
|
433
|
+
A failed or unknown delivery can already have reached the receiver. Deduplicate by the signed delivery ID in durable storage; `payload.id` is the underlying research event ID. Never replay automatically based on a timeout. Changed/recovered events report observations, not independently verified business outcomes. Notifications contain bounded metadata and version identifiers; fetch currently authorized evidence separately with the SDK.
|
|
434
|
+
|
|
435
|
+
Verify signatures **before parsing or normalizing the raw body**:
|
|
436
|
+
|
|
437
|
+
```ts
|
|
438
|
+
import {verifyResearchWebhook} from '@getopencrawl/sdk';
|
|
439
|
+
const verified = await verifyResearchWebhook({
|
|
440
|
+
secret: signingSecret,
|
|
441
|
+
body: rawBytes, // Uint8Array; at most 16,384 bytes
|
|
442
|
+
headers: {
|
|
443
|
+
'webhook-id': request.headers.get('Webhook-Id') ?? '',
|
|
444
|
+
'webhook-timestamp': request.headers.get('Webhook-Timestamp') ?? '',
|
|
445
|
+
'webhook-signature': request.headers.get('Webhook-Signature') ?? '',
|
|
446
|
+
},
|
|
447
|
+
});
|
|
448
|
+
if (!verified) throw new Error('Invalid webhook signature or timestamp');
|
|
449
|
+
// Validate JSON and expected collection, then atomically deduplicate+process verified.deliveryId.
|
|
450
|
+
```
|
|
451
|
+
|
|
452
|
+
The verifier checks HMAC-SHA256 and a five-minute clock tolerance by default; it returns `null` for an invalid signature, timestamp or oversized body. It does not parse payloads, store secrets, or deduplicate deliveries. Keep the receiver clock synchronized. `examples/research-webhook-receiver.mjs` is a bounded Fetch API handler with a required application-owned `processOnce` transaction. It acknowledges only after durable handling; mount it in your server and provide a real persistent deduplication implementation.
|
|
453
|
+
|
|
454
|
+
### Durable customer event receiver (optional Node 24 example)
|
|
455
|
+
|
|
456
|
+
`letter-event-receiver.mjs` connects the verified notification to one retained-event read with the mapped customer's child key. `letter-event-store.mjs` supplies a concrete SQLite inbox and evidence store for **Node 24+**. The general SDK remains Node 22+; this optional adapter uses `node:sqlite`. This example is not an installed Letter integration.
|
|
457
|
+
|
|
458
|
+
```js
|
|
459
|
+
import { mkdirSync } from 'node:fs';
|
|
460
|
+
import { LetterEventStore } from '@getopencrawl/sdk/examples/letter-event-store.mjs';
|
|
461
|
+
import { createLetterEventReceiver } from '@getopencrawl/sdk/examples/letter-event-receiver.mjs';
|
|
462
|
+
|
|
463
|
+
// Server-owned setup, after authorizing the customer and saving its child key.
|
|
464
|
+
// Use your application's private persistent directory, not ephemeral storage.
|
|
465
|
+
mkdirSync('/srv/letter-private', { recursive: true, mode: 0o700 });
|
|
466
|
+
const store = new LetterEventStore('/srv/letter-private/event-inbox.sqlite');
|
|
467
|
+
if (!store.mapping('customer42_research')) store.putMapping({
|
|
468
|
+
id: 'customer42_research', customerId: 'customer42',
|
|
469
|
+
tenantId: savedTenant.id, collectionId: savedChildCollection.id,
|
|
470
|
+
signingSecretRef: 'customer42_webhook', active: true,
|
|
471
|
+
});
|
|
472
|
+
const receive = createLetterEventReceiver({
|
|
473
|
+
store, mappingId: 'customer42_research', baseUrl: 'https://app.getopencrawl.com',
|
|
474
|
+
secrets: customerKeyVault, // get(tenantId): child key; never parent fallback
|
|
475
|
+
webhookSecrets: signingVault, // get(signingSecretRef): this subscription's secret
|
|
476
|
+
});
|
|
477
|
+
// Mount receive(Request) -> Promise<Response> on one fixed HTTPS route in your server.
|
|
478
|
+
```
|
|
479
|
+
|
|
480
|
+
The mapping is local server configuration, not taken from the webhook body. Use a child-created collection: a parent workspace collection ID does not transfer to a child. **Revoking or replacing the provisioning parent key suspends its child credentials and work; replacement does not reconnect them.** Keep the child key and webhook signing secret in encrypted server storage, outside this SQLite database. Changing a local mapping increments its revision and fences in-flight publication; customer/tenant/collection identity cannot be reassigned. Call `putMapping({...mapping, active: false})` to stop that receiver locally.
|
|
481
|
+
|
|
482
|
+
A signed event is bound to the returned collection, observation time, gap state and both version IDs. The receiver reads only the default 16,000-byte source-text window, without following continuation or making a search, crawl, embedding or answer request. It retains both titles, version/body hashes, exact offsets, qualifiers, omitted-context flags and gap/comparison semantics. A partial window cannot independently verify the full-body hashes. The record says `signalClassification: 'not_assessed'`: a vendor wording change, baseline, recovery, unavailable observation or clock update is not automatically a business signal, capability change or buying intent. `webhook.test` records connectivity only.
|
|
483
|
+
|
|
484
|
+
Delivery IDs and semantic `(customer, collection, event)` identities are deduplicated in SQLite across restarts. Acknowledgment follows the durable commit; a lost commit acknowledgment is reconciled before another read. One active lease permits one read, and publication requires its current claim and unchanged local mapping. A later delivery can retry an idempotent GET after a transient failure or expired lease; this is not an exactly-once network guarantee. A 401/403/404 creates a terminal unavailable receipt without text, so duplicate deliveries do not reread denied evidence—even after credentials are repaired or access returns. Recovery of that receipt needs an explicit host-owned operator policy; this example never clears tombstones automatically. No failed request is retried automatically inside the handler.
|
|
485
|
+
|
|
486
|
+
The default caps are 10,000 event records, 10,000 delivery identities and 262,144 bytes per record. A full store returns 503 rather than silently deleting deduplication history. Plan storage capacity, retention and backups in your application; deleting tombstones can admit old deliveries again. `getRecord(customerId, collectionId, eventId)` is a server storage method, not an authorized HTTP endpoint. Check current customer access before exposing saved evidence and enforce your own revocation/erasure policy for those copies. Deactivate mappings before removing customer secrets. Shut down with `store.close()`.
|
|
487
|
+
|
|
488
|
+
Offline checks: `npm test` covers the Node 22-compatible SDK; `npm run test:letter-events` runs the optional Node 24 SQLite receiver suite. CI runs both separately, including packed JavaScript and TypeScript consumers. `readSeerRun` keeps its existing polling/search behavior; this handler is an independent event-driven path.
|
|
489
|
+
|
|
490
|
+
## Hosted package delivery
|
|
491
|
+
|
|
492
|
+
The application image serves the SDK at the canonical URL above. For a local source build and packaging check:
|
|
493
|
+
|
|
494
|
+
```sh
|
|
495
|
+
cd sdk
|
|
496
|
+
npm ci
|
|
497
|
+
npm test
|
|
498
|
+
npm pack
|
|
499
|
+
```
|
|
500
|
+
|
|
501
|
+
The manifest's digest URL identifies exact package bytes and remains available after application upgrades. The rolling `/downloads/opencrawl-sdk.tgz` URL follows the deployed package and is only a compatibility alias. These are hosted artifacts; npm releases of the same package are published as `@getopencrawl/sdk`. Keep the digest URL and integrity in your lockfile, and explicitly choose a new digest when upgrading.
|
|
502
|
+
|
|
503
|
+
## Draft a plan before collecting sources
|
|
504
|
+
|
|
505
|
+
Planning is an optional hosted capability, separate from collection creation. It sends the brief and supplied URLs to Amazon Bedrock; it never crawls a page or creates a collection. Check `client.plans.status()` before offering it.
|
|
506
|
+
|
|
507
|
+
```ts
|
|
508
|
+
const draft = await client.plans.create({
|
|
509
|
+
brief: 'Find public release notes and integration announcements for the companies we follow.',
|
|
510
|
+
seed_urls: ['https://example.com/news'], // optional, up to ten
|
|
511
|
+
}, {idempotencyKey: durablePlanKey});
|
|
512
|
+
// Persist draft.id. Render ready proposal sources for explicit user review.
|
|
513
|
+
// Model suggestions must start unchecked; availability is always 'not_checked'.
|
|
514
|
+
const recovered = await client.plans.byKey(durablePlanKey);
|
|
515
|
+
const current = await client.plans.get(draft.id);
|
|
516
|
+
```
|
|
517
|
+
|
|
518
|
+
The receipt state is `generating`, `ready`, `failed`, `unknown`, or `expired`. A proposal is a suggestion, not evidence. Preserve the distinction between `user_supplied` and `model_suggested` URLs. Only after the user reviews the selected sources and exact allowed origins should the host call `collections.create` with a separate durable key.
|
|
519
|
+
|
|
520
|
+
The SDK can retry the same keyed planning request, but the server never automatically repeats an accepted model invocation. An unknown or expired receipt does not restart; a new draft needs an explicit new key. Proposal content expires after seven days. The server retains key/hash tombstones, imposes concurrent and daily request limits, and never persists a collection from a draft automatically. See [the planning contract](../docs/research-planning.md) for bounds and recovery details.
|
|
521
|
+
|
|
522
|
+
## Embedded customer workspaces
|
|
523
|
+
|
|
524
|
+
Use a parent **administrator API key** only inside your server to provision isolated customer research workspaces. A parent key manages delegation metadata; it does not read the customer's evidence. Authorize the host customer and look up its server-stored tenant mapping before selecting any credential. Never trust a tenant ID supplied by an unauthenticated browser.
|
|
525
|
+
|
|
526
|
+
**Keep the exact provisioning parent key active.** Revoking, expiring or replacing it suspends its child credentials and background jobs. Rotating a child key does not transfer this dependency to a new parent key. Choose a dedicated server provisioning credential before creating customers; do not use a temporary test key for lasting customer workspaces.
|
|
527
|
+
|
|
528
|
+
```ts
|
|
529
|
+
const baseUrl = 'https://app.getopencrawl.com';
|
|
530
|
+
const parent = new OpenCrawl({baseUrl, apiKey: serverSecrets.parentAdminKey});
|
|
531
|
+
// Persist this key and exact descriptor before sending. This method sends once.
|
|
532
|
+
const created = await parent.createServiceTenant({
|
|
533
|
+
external_customer_id: authorizedCustomer.id, // stable internal ID, not email
|
|
534
|
+
name: authorizedCustomer.displayName,
|
|
535
|
+
}, {idempotencyKey: durableCreateKey});
|
|
536
|
+
if (created.key !== null) await secretStore.put(created.tenant.id, created.key);
|
|
537
|
+
await mappingStore.save(authorizedCustomer.id, created.tenant.id, created.receipt);
|
|
538
|
+
|
|
539
|
+
// Separate instance and credential for this customer's collections and evidence.
|
|
540
|
+
const child = new OpenCrawl({baseUrl, apiKey: await secretStore.get(created.tenant.id)});
|
|
541
|
+
const collections = await child.collections.list();
|
|
542
|
+
```
|
|
543
|
+
|
|
544
|
+
Store the first-response-only key in durable encrypted server storage before acknowledging setup. Do not log mutation responses. No SDK method stores a secret or customer mapping for you. `examples/letter-service-tenants.mjs` supplies small server helpers and a runnable offline demonstration:
|
|
545
|
+
|
|
546
|
+
```sh
|
|
547
|
+
node examples/letter-service-tenants.mjs --mock
|
|
548
|
+
```
|
|
549
|
+
|
|
550
|
+
The six typed methods are:
|
|
551
|
+
|
|
552
|
+
- `createServiceTenant({external_customer_id,name}, {idempotencyKey})` → tenant, acceptance receipt and first-response key.
|
|
553
|
+
- `listServiceTenants({external_customer_id?,limit?,offset?})` → `{items,next_offset}`; pass the returned offset explicitly for the next page.
|
|
554
|
+
- `getServiceTenant(tenantId)` → current metadata and effective `access_state` (`active`, `suspended`, or `revoked`).
|
|
555
|
+
- `rotateServiceTenantKey(tenantId, {expected_revision}, {idempotencyKey})` → new first-response key; invalidates the old child key.
|
|
556
|
+
- `revokeServiceTenant(tenantId, {expected_revision}, {idempotencyKey})` → terminal revocation; pauses sources/cancels ordinary work but **does not erase retained data**.
|
|
557
|
+
- `getServiceTenantReceipt('create' | 'rotate:<tenantId>' | 'revoke:<tenantId>', {idempotencyKey})` → read-only recovery using the original key in a header, never in the URL.
|
|
558
|
+
|
|
559
|
+
Create, rotate and revoke **never retry automatically**, even if the client has `maxRetries` enabled. A lost response can represent an accepted change. Read its receipt before deciding what to do next. A recovered response has `key:null`; its historical `target_revision` and the tenant's current revision can differ. If the first key was lost and is absent from your secret store, read the current tenant and explicitly rotate with that revision and a **new saved operation key**. Never create another customer or rotate automatically after a timeout.
|
|
560
|
+
|
|
561
|
+
The delegation stays anchored to the exact original parent key and its administrator. Parent-key revocation/expiry, child-key expiry, administrator access loss or customer revocation suspends downstream authority, including background work; rotating the child does not transfer the parent anchor. Child credentials cannot provision tenants, create general API keys, access sibling workspaces, or use the hosted research planner. Use the child instance only for its allowed research/evidence operations. A locally `active` metadata response is not a health guarantee; actual child requests perform current authorization checks.
|
|
562
|
+
|
|
563
|
+
For optional requested-answer assessments on a saved run, use
|
|
564
|
+
`const assessment = await readResearchAnswerContract(run)` (import it from
|
|
565
|
+
`@getopencrawl/sdk`). Keep all `run.gaps`; append `assessment.notices` by their
|
|
566
|
+
`text` field. The accessor suppresses only an exact, hash-bound notice already
|
|
567
|
+
present in a gap. `assessment.contract` retains the original goal hash, exact
|
|
568
|
+
request fragments, source-window scope and links to surviving literal fields.
|
|
569
|
+
The goal hash and excerpts are server provenance: a run does not contain the
|
|
570
|
+
original full goal for a client to recompute that binding. Never substitute the
|
|
571
|
+
agent’s current, editable goal for a historical run.
|
|
572
|
+
A source-scoped unknown means the saved windows establish neither an answer nor
|
|
573
|
+
its contradiction. `available_but_not_delivered` means answer material was found
|
|
574
|
+
but a complete answer was not retained. Missing, invalid or unassessed metadata
|
|
575
|
+
never establishes completeness. This optional reader makes no requests, changes
|
|
576
|
+
no run status and does not verify factual truth. Raw run JSON remains unchanged.
|
|
577
|
+
|
|
578
|
+
The optional concise answer has a separate accessor:
|
|
579
|
+
`const answer = await readResearchAnswerView(run)`. A `valid` result contains
|
|
580
|
+
question-ordered `sections` with a validated `component`, reviewed `assertions`,
|
|
581
|
+
and an `omission_reason`/fixed `notice` when prose was not retained. Each assertion
|
|
582
|
+
keeps its exact `fact_refs`; use those to reveal the original findings,
|
|
583
|
+
qualifications and citations. Keep all original gaps and coverage notices too.
|
|
584
|
+
Absent or invalid metadata returns no sections, so use the ordinary evidence
|
|
585
|
+
presentation. The reader checks the saved contract, full-findings digest and
|
|
586
|
+
literal references; it does not independently verify the prose's factual meaning
|
|
587
|
+
or certify completeness. No calls or rewriting occur. This optional source
|
|
588
|
+
capability needs a matching deployed release; old runs are never regenerated.
|
|
589
|
+
|
|
590
|
+
## Private product-screen evidence (source feature)
|
|
591
|
+
|
|
592
|
+
With a matching server/client release, `client.productMonitors` provides typed monitor settings, capture receipts, immutable snapshot/image reads, exact before/after comparisons and bounded text context. Browser login/capture belongs to the local CLI; this SDK never uploads browser cookies or storage. `image(snapshotId)` verifies bounded PNG bytes against the immutable snapshot hash. `comparison(comparisonId, version)` refuses a different version.
|
|
593
|
+
|
|
594
|
+
Create and capture use caller-persisted UUIDs and never automatically retry. After an uncertain write, use `byClientId` or `captureByClientId` with the original ID; do not generate another request. Settings updates require `expected_generation`. See [local capture, recovery and evidence limits](../docs/product-monitoring-cli.md). This source implementation and its synthetic tests do not establish hosted availability.
|
|
595
|
+
|
|
596
|
+
|
|
597
|
+
For an operator-enabled final presentation run, explicitly select `completion_v2` in the same `research_policy` field (or CLI manual-run `--research-policy completion_v2`). Omitting the field keeps the existing request unchanged. The optional `usage.presentation` record reports bounded presentation status and known/uncertain usage; it does not certify completeness or replace findings, gaps, quotes, or `usage.answer_view` validation. SDK/shared readers expose it through `readResearchPresentation(run)` and ignore malformed optional metadata.
|
|
598
|
+
|
|
599
|
+
For a validated `completion_v2` receipt, the default research-agent read wait ends 540 seconds after that wait began, allowing 480 seconds of server work plus settlement time. Repeated polls never renew it. An explicit SDK `maxWaitMs` or CLI `--timeout` still takes precedence; ordinary/v1 waits remain 300 seconds and individual default GET requests remain capped at 30 seconds. Durable customer-research poll batches and operation leases are unchanged. A timeout only stops local waiting: continue reading the same run, without another admission.
|