@kontourai/lookout 0.3.0 → 0.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +189 -32
- package/dist/src/check-result.d.ts +2 -2
- package/dist/src/check-runner.d.ts +1 -1
- package/dist/src/check-runner.js +39 -26
- package/dist/src/drift-emission.js +2 -0
- package/dist/src/index.d.ts +8 -2
- package/dist/src/index.js +3 -1
- package/dist/src/observe-extract-diff.d.ts +90 -0
- package/dist/src/observe-extract-diff.js +154 -0
- package/dist/src/proposal-diff.d.ts +4 -0
- package/dist/src/proposal-diff.js +49 -4
- package/dist/src/registry.d.ts +16 -4
- package/dist/src/registry.js +107 -41
- package/dist/src/semantic-review-projection.d.ts +13 -0
- package/dist/src/semantic-review-projection.js +68 -0
- package/dist/src/semantic-review-types.d.ts +95 -0
- package/dist/src/semantic-review-types.js +1 -0
- package/dist/src/semantic-review-work.d.ts +5 -0
- package/dist/src/semantic-review-work.js +36 -0
- package/dist/src/snapshot-store.d.ts +11 -1
- package/dist/src/snapshot-store.js +18 -1
- package/package.json +5 -4
package/README.md
CHANGED
|
@@ -3,12 +3,58 @@
|
|
|
3
3
|
**Cheap, honest drift detection for content you re-check over time — "did this source change since we last looked?"**
|
|
4
4
|
|
|
5
5
|
A small **source registry** and **non-throwing drift-check runner** built on
|
|
6
|
-
[
|
|
7
|
-
|
|
6
|
+
[Forage](https://github.com/kontourai/forage) snapshots. It composes Forage for
|
|
7
|
+
fetching and snapshot storage, provides deterministic proposal diffing, and
|
|
8
8
|
emits neutral, typed drift — its own vocabulary, in its own dependency-free
|
|
9
|
-
package. It does **not**
|
|
10
|
-
Surface, review claims, notify, crawl, or schedule.
|
|
11
|
-
returned as typed results.
|
|
9
|
+
package. It does **not** implement acquisition or extraction, author trust-layer
|
|
10
|
+
records, project to Surface, review claims, notify, crawl, or schedule.
|
|
11
|
+
Operational failures are returned as typed results.
|
|
12
|
+
|
|
13
|
+
## Quick start
|
|
14
|
+
|
|
15
|
+
Register one source in a JSON file (defaults to `<cwd>/lookout.sources.json`;
|
|
16
|
+
see [Registry](#registry) for every field):
|
|
17
|
+
|
|
18
|
+
```json
|
|
19
|
+
{
|
|
20
|
+
"version": 1,
|
|
21
|
+
"sources": [
|
|
22
|
+
{
|
|
23
|
+
"id": "example-home",
|
|
24
|
+
"kind": "web-page",
|
|
25
|
+
"url": "https://example.com/",
|
|
26
|
+
"targetSchema": [{ "path": "title", "type": "string", "required": true }],
|
|
27
|
+
"cadenceHint": "daily",
|
|
28
|
+
"renderPolicy": "never"
|
|
29
|
+
}
|
|
30
|
+
]
|
|
31
|
+
}
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
Then check it with the CLI:
|
|
35
|
+
|
|
36
|
+
```bash
|
|
37
|
+
lookout check example-home --registry ./lookout.sources.json
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
The first run has nothing to compare against, so it persists a baseline
|
|
41
|
+
snapshot and reports `changed`/`initial`:
|
|
42
|
+
|
|
43
|
+
```json
|
|
44
|
+
{"sourceId":"example-home","sourceUrl":"https://example.com/","checkedAt":"2026-07-20T14:12:48.248Z","warnings":[],"kind":"changed","priorSnapshotRef":null,"currentSnapshotRef":"forage-snapshot:example-home?...","changeBasis":"initial"}
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
Run the same command again and, if the source hasn't changed, Forage's
|
|
48
|
+
conditional request returns `304` with zero body transfer — no re-download,
|
|
49
|
+
just confirmation:
|
|
50
|
+
|
|
51
|
+
```json
|
|
52
|
+
{"sourceId":"example-home","sourceUrl":"https://example.com/","checkedAt":"2026-07-20T14:12:53.089Z","warnings":[],"kind":"unchanged-304","snapshotRef":"forage-snapshot:example-home?..."}
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
Exit code is `0` for both runs — a per-source result, even a fetch failure, is
|
|
56
|
+
not a CLI failure. See [CLI](#cli) for the full exit-code contract and
|
|
57
|
+
[Check results](#check-results) for what each `kind` means.
|
|
12
58
|
|
|
13
59
|
## Why it's different
|
|
14
60
|
|
|
@@ -21,8 +67,8 @@ as "no change"). Lookout is the opposite on all three:
|
|
|
21
67
|
|---|---|---|
|
|
22
68
|
| Cost | full re-download every check | **conditional `304`** — often no download at all |
|
|
23
69
|
| Signal | raw bytes (ads/timestamps = "changed") | **proposal-identity diff** — "a *new entity appeared*" vs "a byte moved" |
|
|
24
|
-
| Honesty | a crash or false-`304` silently reads as "unchanged" | **typed results, never throws**;
|
|
25
|
-
| Output | your problem to shape | **neutral, typed drift** — events already Hachure-evidence-shaped, ready for a consumer to lift into a trust bundle |
|
|
70
|
+
| Honesty | a crash or false-`304` silently reads as "unchanged" | **typed results, never throws**; Forage scopes validators to the exact prior resource and Lookout verifies the returned capture identity |
|
|
71
|
+
| Output | your problem to shape | **neutral, typed drift** — events already [Hachure](https://github.com/hachure-org/spec)-evidence-shaped (the open, product-neutral trust-record spec Surface's TrustBundle implements), ready for a consumer to lift into a trust bundle |
|
|
26
72
|
|
|
27
73
|
So the point isn't "diffing" — it's *cheap + honest + semantic + review-ready*
|
|
28
74
|
change detection, so a periodic re-check surfaces **only the real delta** (this
|
|
@@ -37,8 +83,7 @@ One layer in a four-verb stack; each repo owns one verb and is usable alone:
|
|
|
37
83
|
- **lookout** — CHANGE: did this registered source drift since last look?
|
|
38
84
|
- **survey** — the SHAPE: what reviewed truth looks like (claims / review / resolution)
|
|
39
85
|
|
|
40
|
-
Lookout composes
|
|
41
|
-
`forage` as its single-source fetch surface lands) for cheap `304`-aware re-checks,
|
|
86
|
+
Lookout composes Forage's `/fetch` surface for cheap `304`-aware re-checks,
|
|
42
87
|
and Traverse's `ExtractionProposal` identity for the semantic diff. Dependency
|
|
43
88
|
arrows point only downward — no cycles. Lookout itself depends on nothing in the
|
|
44
89
|
trust layer: its events are already Hachure-evidence-shaped (`snapshotRef` /
|
|
@@ -58,12 +103,11 @@ owns its transport and opts out of the default guard.
|
|
|
58
103
|
## Requirements
|
|
59
104
|
|
|
60
105
|
- Node.js `>= 22`.
|
|
61
|
-
- `@kontourai/
|
|
62
|
-
trustworthy `unchanged-304`
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
different URL and report a **false** `304`, so do not downgrade.
|
|
106
|
+
- `@kontourai/forage` `>= 0.4.0`. Exact durable-reference replay requires the
|
|
107
|
+
resolver first released in `0.4.0`; trustworthy `unchanged-304`
|
|
108
|
+
classification also depends on Forage's validator-scoped revalidation.
|
|
109
|
+
Lookout exact-pins Traverse separately for schema and extraction-proposal
|
|
110
|
+
contracts.
|
|
67
111
|
|
|
68
112
|
## Registry
|
|
69
113
|
|
|
@@ -81,6 +125,13 @@ override with the library path argument or the CLI `--registry` flag):
|
|
|
81
125
|
"targetSchema": [{ "path": "title", "type": "string", "required": true }],
|
|
82
126
|
"cadenceHint": "daily",
|
|
83
127
|
"renderPolicy": "never"
|
|
128
|
+
},
|
|
129
|
+
{
|
|
130
|
+
"id": "published-results",
|
|
131
|
+
"kind": "structured-file",
|
|
132
|
+
"format": "yaml",
|
|
133
|
+
"url": "https://raw.githubusercontent.com/example/project/0123456789abcdef0123456789abcdef01234567/results.yml",
|
|
134
|
+
"cadenceHint": "weekly"
|
|
84
135
|
}
|
|
85
136
|
]
|
|
86
137
|
}
|
|
@@ -91,11 +142,17 @@ Fields per source:
|
|
|
91
142
|
| Field | Meaning |
|
|
92
143
|
| --- | --- |
|
|
93
144
|
| `id` | Non-empty, unique within the document. Used for exact lookup and snapshot identity. |
|
|
94
|
-
| `kind` | `web-page` or `
|
|
145
|
+
| `kind` | `web-page`, `api-record`, or `structured-file`. |
|
|
95
146
|
| `url` | Absolute HTTP(S) URL. |
|
|
96
|
-
| `
|
|
147
|
+
| `format` | Required only for `structured-file`: `yaml`, `json`, or `csv`. Lookout does not parse it. |
|
|
148
|
+
| `targetSchema` | Required for `web-page` and `api-record`: Traverse `TargetFieldSchema[]`. Stored inert at L1 (no extraction yet). |
|
|
97
149
|
| `cadenceHint` | Non-empty string; advisory only — Lookout does not schedule. |
|
|
98
|
-
| `renderPolicy` | `never` \| `on-shell-warning` \| `always`. **Inert** at L1
|
|
150
|
+
| `renderPolicy` | Required for `web-page` and `api-record`: `never` \| `on-shell-warning` \| `always`. **Inert** at L1. |
|
|
151
|
+
|
|
152
|
+
Structured-file entries retain raw fetched bytes and use the same guarded fetch,
|
|
153
|
+
snapshot, comparison, and replay path as every other source. Parsing and domain
|
|
154
|
+
normalization stay downstream. Prefer immutable, commit-pinned artifact URLs so
|
|
155
|
+
an upstream branch move cannot silently redefine the cited source location.
|
|
99
156
|
|
|
100
157
|
Validation reports **every** deterministic issue at once (with index/id context)
|
|
101
158
|
and never reads the network. Lookup is exact-id; listing preserves file order —
|
|
@@ -104,35 +161,39 @@ no merging, discovery, or remote registries.
|
|
|
104
161
|
## Check results
|
|
105
162
|
|
|
106
163
|
A check classifies each source into exactly one of four kinds. Every result
|
|
107
|
-
carries `sourceId`, the registered `sourceUrl`, `checkedAt`, and
|
|
108
|
-
|
|
164
|
+
carries `sourceId`, the registered `sourceUrl`, `checkedAt`, and Forage fetch
|
|
165
|
+
warnings.
|
|
109
166
|
|
|
110
167
|
| `kind` | When | Extra fields |
|
|
111
168
|
| --- | --- | --- |
|
|
112
169
|
| `unchanged-304` | A validator-backed conditional request returned `304`; zero body transfer; the prior snapshot is not re-persisted. | `snapshotRef` |
|
|
113
170
|
| `unchanged-hash` | A full body was fetched and persisted, but its sha256 `bodyHash` equals the prior, **and** it came from the same resource URL — established by Lookout's own comparison. | `priorSnapshotRef`, `currentSnapshotRef` |
|
|
114
171
|
| `changed` | The fresh body differs from the prior (`changeBasis: "hash"`), **or** it is the first successful observation (`changeBasis: "initial"`, `priorSnapshotRef: null`). | `priorSnapshotRef` (nullable), `currentSnapshotRef`, `changeBasis` |
|
|
115
|
-
| `error` | Any operational failure — contained so the runner never rejects. | `origin` (`
|
|
172
|
+
| `error` | Any operational failure — contained so the runner never rejects. | `origin` (`forage` \| `lookout`), `error` |
|
|
116
173
|
|
|
117
174
|
`unchanged-hash` asserts same-resource continuity, so it requires **both** an
|
|
118
175
|
identical `bodyHash` **and** the same resource URL as the prior snapshot. If a
|
|
119
176
|
source has moved — the prior snapshot's URL differs from this fetch's — the
|
|
120
177
|
result is `changed` even when the bytes are byte-identical, and the fresh
|
|
121
178
|
snapshot is persisted as the new baseline. (The `unchanged-304` path is already
|
|
122
|
-
resource-scoped by
|
|
179
|
+
resource-scoped by Forage's validators, so only the hash path needs this
|
|
123
180
|
guard.)
|
|
124
181
|
|
|
125
|
-
`error` results preserve provenance: `origin: "
|
|
182
|
+
`error` results preserve provenance: `origin: "forage"` carries Forage's
|
|
126
183
|
discriminated `FetchError` verbatim (its `kind`, and `status` when present);
|
|
127
184
|
`origin: "lookout"` carries a typed `kind` — `prior-read`, `persistence`,
|
|
128
185
|
`dependency-contract`, or `unexpected`.
|
|
129
186
|
|
|
130
187
|
Snapshot references (`snapshotRef` / `priorSnapshotRef` / `currentSnapshotRef`)
|
|
131
|
-
are portable logical refs from
|
|
132
|
-
filesystem paths. Snapshots are stored via
|
|
188
|
+
are portable logical refs from Forage's `buildSnapshotSourceRef` — never
|
|
189
|
+
filesystem paths. Snapshots are stored via Forage's filesystem store, rooted by
|
|
133
190
|
default at `<cwd>/.kontourai/lookout/snapshots` (override with the CLI
|
|
134
191
|
`--snapshot-root` flag or by injecting a store in library use). Lookout adds no
|
|
135
|
-
custom filenames or retention.
|
|
192
|
+
custom filenames or retention. `resolveLookoutSnapshot()` replays one exact
|
|
193
|
+
reference through an injected store or Lookout snapshot root, authenticates its
|
|
194
|
+
body and replay metadata, and never fetches. References emitted before Forage's
|
|
195
|
+
replay-envelope digest remain resolvable with `integrity: "body-and-identity"`;
|
|
196
|
+
new references report `integrity: "snapshot-envelope"`.
|
|
136
197
|
|
|
137
198
|
## Schema coverage
|
|
138
199
|
|
|
@@ -144,6 +205,7 @@ can't catch this. `checkSchemaCoverage` is the static complement:
|
|
|
144
205
|
```ts
|
|
145
206
|
import { checkSchemaCoverage } from "@kontourai/lookout";
|
|
146
207
|
|
|
208
|
+
if (source.kind === "structured-file") throw new Error("structured sources are parsed downstream");
|
|
147
209
|
const { covered, gaps } = checkSchemaCoverage(source.targetSchema, proposals);
|
|
148
210
|
// covered: declared paths at least one proposal produced (schema order)
|
|
149
211
|
// gaps: declared paths that produced none, each with its `required` flag
|
|
@@ -213,17 +275,112 @@ const results = await runner.checkAll(registry.list());
|
|
|
213
275
|
```
|
|
214
276
|
|
|
215
277
|
`createCheckRunner` accepts injected seams — `store`, `fetchSource` (defaults to
|
|
216
|
-
|
|
278
|
+
Forage's SSRF-guarded fetcher), `fetchOptions`, and
|
|
217
279
|
`clock` — so checks run with no live network or timers in tests. Injecting either
|
|
218
280
|
`fetchSource` or `fetchOptions.fetch` overrides the default guarded transport.
|
|
219
281
|
|
|
282
|
+
## Observe changed sources without re-extracting unchanged ones
|
|
283
|
+
|
|
284
|
+
`createObserveExtractDiff` is an optional library composition for callers that
|
|
285
|
+
want one typed observation per check. Supply acquisition, extraction, and
|
|
286
|
+
recording capabilities; Lookout does not choose a content-preparation method,
|
|
287
|
+
provider, or observation database.
|
|
288
|
+
|
|
289
|
+
```ts
|
|
290
|
+
import { createObserveExtractDiff } from "@kontourai/lookout";
|
|
291
|
+
|
|
292
|
+
const composition = createObserveExtractDiff({
|
|
293
|
+
acquisition: { check: runner.check },
|
|
294
|
+
extraction: {
|
|
295
|
+
async extract({ source, snapshotRef }) {
|
|
296
|
+
// Resolve `snapshotRef`, prepare content, and invoke a caller-selected
|
|
297
|
+
// extraction implementation. Return its public Traverse result.
|
|
298
|
+
return extractChangedSource(source, snapshotRef);
|
|
299
|
+
},
|
|
300
|
+
},
|
|
301
|
+
recorder: {
|
|
302
|
+
async record(observation) {
|
|
303
|
+
// Caller-owned continuity and durable storage.
|
|
304
|
+
return saveObservation(observation);
|
|
305
|
+
},
|
|
306
|
+
},
|
|
307
|
+
});
|
|
308
|
+
|
|
309
|
+
const result = await composition.observe(source);
|
|
310
|
+
```
|
|
311
|
+
|
|
312
|
+
`unchanged-304` and `unchanged-hash` are recorded as `unchanged` and never call
|
|
313
|
+
the extraction capability, so they make zero preparation and provider calls.
|
|
314
|
+
Changed observations retain source and snapshot references, Traverse's prepared
|
|
315
|
+
artifact identity, the proposal set, and the current/prior observation
|
|
316
|
+
identities returned by the recorder. `partial`, `provider-failure`, mixed
|
|
317
|
+
`partial-provider-failure`, and `extraction-failure` remain distinct typed
|
|
318
|
+
outcomes. Provider failures are reduced to provider-neutral `kind` and
|
|
319
|
+
`retryable` classifications; provider names, messages, native diagnostics,
|
|
320
|
+
free-form extraction warnings, raw responses, and thrown-error text are not
|
|
321
|
+
copied into the durable observation.
|
|
322
|
+
A first changed observation
|
|
323
|
+
has `priorObservationId: null`; it is a baseline observation, not a fabricated
|
|
324
|
+
list of additions or removals.
|
|
325
|
+
|
|
326
|
+
The composition validates that the check identifies the requested registered
|
|
327
|
+
source and that Traverse validates the prepared artifact whose snapshot anchor
|
|
328
|
+
matches the current check. This prevents mismatched capability results from
|
|
329
|
+
being recorded as one observation. Proposal excerpts, source URLs, and
|
|
330
|
+
acquisition check warnings can still contain sensitive source material; callers
|
|
331
|
+
must apply their own retention and access policy before durable storage.
|
|
332
|
+
|
|
333
|
+
The existing `createCheckRunner` and `createDriftEmitter` entrypoints remain
|
|
334
|
+
available. Use `createDriftEmitter` when the caller wants deterministic
|
|
335
|
+
proposal-diff events with its own identity callbacks.
|
|
336
|
+
|
|
337
|
+
## Route semantic transitions to review
|
|
338
|
+
|
|
339
|
+
`buildSemanticReviewWork` turns one genuine prior/current proposal transition
|
|
340
|
+
into structurally Survey-compatible `ReviewItem` resources without adding a
|
|
341
|
+
runtime dependency on a review product. The caller supplies entity/field
|
|
342
|
+
identity callbacks and claim meaning; Lookout supplies deterministic transition
|
|
343
|
+
and item identities.
|
|
344
|
+
|
|
345
|
+
```ts
|
|
346
|
+
const work = buildSemanticReviewWork({
|
|
347
|
+
prior,
|
|
348
|
+
current,
|
|
349
|
+
observationIdentity: { prior: priorId, current: currentId },
|
|
350
|
+
schema: source.targetSchema,
|
|
351
|
+
selectEntities,
|
|
352
|
+
entityIdentity,
|
|
353
|
+
proposalsFor,
|
|
354
|
+
fieldIdentity,
|
|
355
|
+
claimTarget(change) {
|
|
356
|
+
return describeClaim(change);
|
|
357
|
+
},
|
|
358
|
+
});
|
|
359
|
+
```
|
|
360
|
+
|
|
361
|
+
Added, removed, moved, provenance-changed, and value-changed proposals become distinct work items.
|
|
362
|
+
New coverage or exact-provenance gaps are also reviewable. Each available side
|
|
363
|
+
retains its exact snapshot reference, observation time, locator, excerpt, and
|
|
364
|
+
extractor. An absent side is explicit and anchored to the corresponding
|
|
365
|
+
observation snapshot, so a removal remains reviewable even when the new
|
|
366
|
+
extraction emits no value. Identical proposal sets produce no semantic work;
|
|
367
|
+
replaying the same pair of observation identities produces byte-identical
|
|
368
|
+
resources.
|
|
369
|
+
|
|
370
|
+
Review resolution, escalation, persistence, and authority policy stay with the
|
|
371
|
+
consumer. Evidence excerpts and caller-supplied claim targets can contain
|
|
372
|
+
sensitive data; apply retention, redaction, and access controls before storing
|
|
373
|
+
or forwarding review work. Only compact typed identities cross the producer
|
|
374
|
+
metadata boundary; no provider messages, native diagnostics, or raw responses
|
|
375
|
+
are copied.
|
|
376
|
+
|
|
220
377
|
## Non-goals
|
|
221
378
|
|
|
222
|
-
|
|
223
|
-
authority policy, rendered-fetch wiring
|
|
224
|
-
(`renderPolicy` is inert
|
|
225
|
-
|
|
226
|
-
|
|
379
|
+
Acquisition and extraction implementations, Surface projection, notifications,
|
|
380
|
+
crawling, review/escalation or authority policy, rendered-fetch wiring
|
|
381
|
+
(`renderPolicy` is inert), and scheduling. Forage retains all fetch politeness,
|
|
382
|
+
robots, redirects, retries, timeouts, headers, user-agent, and rendered-fetch
|
|
383
|
+
behavior.
|
|
227
384
|
|
|
228
385
|
## Development
|
|
229
386
|
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { FetchError } from "@kontourai/
|
|
1
|
+
import type { FetchError } from "@kontourai/forage/fetch";
|
|
2
2
|
export interface CheckResultCommon {
|
|
3
3
|
sourceId: string;
|
|
4
4
|
sourceUrl: string;
|
|
@@ -23,7 +23,7 @@ export interface ChangedResult extends CheckResultCommon {
|
|
|
23
23
|
export type LookoutErrorKind = "prior-read" | "persistence" | "dependency-contract" | "unexpected";
|
|
24
24
|
export interface ErrorResult extends CheckResultCommon {
|
|
25
25
|
kind: "error";
|
|
26
|
-
origin: "
|
|
26
|
+
origin: "forage" | "lookout";
|
|
27
27
|
error: FetchError | {
|
|
28
28
|
kind: LookoutErrorKind;
|
|
29
29
|
message: string;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { FetchResult, FetchSourceOptions, SnapshotStore, SourceConfig } from "@kontourai/
|
|
1
|
+
import type { FetchResult, FetchSourceOptions, SnapshotStore, SourceConfig } from "@kontourai/forage/fetch";
|
|
2
2
|
import type { CheckResult } from "./check-result.js";
|
|
3
3
|
import type { ProviderResolver } from "./provider-resolution.js";
|
|
4
4
|
import type { LookoutSource } from "./registry.js";
|
package/dist/src/check-runner.js
CHANGED
|
@@ -1,30 +1,20 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
1
|
+
import { isDeepStrictEqual } from "node:util";
|
|
2
|
+
import { buildSnapshotSourceRef, fetchSource as forageFetchSource, } from "@kontourai/forage/fetch";
|
|
3
3
|
/**
|
|
4
|
-
* The
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
4
|
+
* The egress policy for lookout's registered-source fetches. Registered source
|
|
5
|
+
* URLs are operator- / aggregator-supplied and not fully trusted, so a source
|
|
6
|
+
* pointing at a private, link-local, loopback, or cloud-metadata host must be
|
|
7
|
+
* refused before any connection — a drift check can never be turned into an
|
|
8
|
+
* SSRF vector. `forage`'s `fetchSource` builds its own SSRF-pinned guarded
|
|
9
|
+
* transport from this policy whenever the caller doesn't inject a custom
|
|
10
|
+
* `fetch` (e.g. tests), so this is the single shared policy literal — never
|
|
11
|
+
* scattered per call site, never `{ guarded: false }`.
|
|
12
12
|
*/
|
|
13
|
-
|
|
14
|
-
function defaultGuardedFetch() {
|
|
15
|
-
cachedGuardedFetch ??= createGuardedFetch();
|
|
16
|
-
return cachedGuardedFetch;
|
|
17
|
-
}
|
|
13
|
+
const GUARDED_EGRESS = { guarded: true };
|
|
18
14
|
export function createCheckRunner(options) {
|
|
19
|
-
const
|
|
20
|
-
const fetchImpl = options.fetchSource ?? traverseFetchSource;
|
|
15
|
+
const fetchImpl = options.fetchSource ?? forageFetchSource;
|
|
21
16
|
const clock = options.clock ?? (() => new Date().toISOString());
|
|
22
|
-
// Guard the default traverse fetcher's egress with forage. A caller-supplied
|
|
23
|
-
// fetcher or `fetchOptions.fetch` owns its own transport and is left as-is.
|
|
24
17
|
const fetchOptions = { ...options.fetchOptions };
|
|
25
|
-
if (usingDefaultFetcher && fetchOptions.fetch === undefined) {
|
|
26
|
-
fetchOptions.fetch = defaultGuardedFetch();
|
|
27
|
-
}
|
|
28
18
|
async function check(source) {
|
|
29
19
|
const common = () => ({
|
|
30
20
|
sourceId: source.id,
|
|
@@ -41,17 +31,17 @@ export function createCheckRunner(options) {
|
|
|
41
31
|
}
|
|
42
32
|
let fetched;
|
|
43
33
|
try {
|
|
44
|
-
fetched = await fetchImpl({ id: source.id, url: source.url,
|
|
34
|
+
fetched = await fetchImpl({ id: source.id, url: source.url, egress: GUARDED_EGRESS }, { ...fetchOptions, store: options.store });
|
|
45
35
|
}
|
|
46
36
|
catch (error) {
|
|
47
37
|
return lookoutError(common(), "unexpected", error);
|
|
48
38
|
}
|
|
49
39
|
const base = { ...common(), warnings: Array.isArray(fetched?.warnings) ? [...fetched.warnings] : [] };
|
|
50
40
|
if (!isFetchResult(fetched)) {
|
|
51
|
-
return lookoutError(base, "dependency-contract", "
|
|
41
|
+
return lookoutError(base, "dependency-contract", "Forage returned neither exactly one snapshot nor exactly one error");
|
|
52
42
|
}
|
|
53
43
|
if (fetched.error) {
|
|
54
|
-
return { ...base, kind: "error", origin: "
|
|
44
|
+
return { ...base, kind: "error", origin: "forage", error: fetched.error };
|
|
55
45
|
}
|
|
56
46
|
// Defense-in-depth: even if a future guard gap let a malformed snapshot
|
|
57
47
|
// through, any stray throw in classification/ref-building becomes a typed
|
|
@@ -59,7 +49,22 @@ export function createCheckRunner(options) {
|
|
|
59
49
|
try {
|
|
60
50
|
const snapshot = fetched.snapshot;
|
|
61
51
|
if (snapshot.notModified === true) {
|
|
62
|
-
|
|
52
|
+
const snapshotBodyEncoding = bodyEncoding(snapshot);
|
|
53
|
+
const priorBodyEncoding = prior === undefined ? undefined : bodyEncoding(prior);
|
|
54
|
+
if (prior === undefined ||
|
|
55
|
+
snapshotBodyEncoding === undefined ||
|
|
56
|
+
snapshotBodyEncoding !== priorBodyEncoding ||
|
|
57
|
+
snapshot.sourceId !== prior.sourceId ||
|
|
58
|
+
snapshot.url !== prior.url ||
|
|
59
|
+
snapshot.status !== prior.status ||
|
|
60
|
+
snapshot.fetchedAt !== prior.fetchedAt ||
|
|
61
|
+
snapshot.bodyHash !== prior.bodyHash ||
|
|
62
|
+
!isDeepStrictEqual(snapshot.headers, prior.headers) ||
|
|
63
|
+
!isDeepStrictEqual(snapshot.redirects, prior.redirects) ||
|
|
64
|
+
snapshot.rendered !== prior.rendered) {
|
|
65
|
+
return lookoutError(base, "dependency-contract", "Forage returned a 304 snapshot without the matching prior capture");
|
|
66
|
+
}
|
|
67
|
+
return { ...base, kind: "unchanged-304", snapshotRef: buildSnapshotSourceRef(prior) };
|
|
63
68
|
}
|
|
64
69
|
try {
|
|
65
70
|
await options.store.put(snapshot);
|
|
@@ -99,6 +104,14 @@ export function createCheckRunner(options) {
|
|
|
99
104
|
}
|
|
100
105
|
return { check, checkAll };
|
|
101
106
|
}
|
|
107
|
+
function bodyEncoding(snapshot) {
|
|
108
|
+
const descriptor = Object.getOwnPropertyDescriptor(snapshot, "body");
|
|
109
|
+
if (descriptor === undefined || !("value" in descriptor))
|
|
110
|
+
return undefined;
|
|
111
|
+
if (typeof descriptor.value === "string")
|
|
112
|
+
return "utf8";
|
|
113
|
+
return descriptor.value instanceof Uint8Array ? "bytes" : undefined;
|
|
114
|
+
}
|
|
102
115
|
function isFetchResult(value) {
|
|
103
116
|
if (typeof value !== "object" || value === null)
|
|
104
117
|
return false;
|
|
@@ -16,6 +16,8 @@ function normalizeDiff(value) {
|
|
|
16
16
|
removedProposalOccurrences: sorted(value.facts.removedProposalOccurrences),
|
|
17
17
|
provenanceChanges: sorted(value.facts.provenanceChanges),
|
|
18
18
|
removedEntities: [...value.facts.removedEntities].sort(),
|
|
19
|
+
addedProposalEvidence: sorted(value.facts.addedProposalEvidence ?? []),
|
|
20
|
+
removedProposalEvidence: sorted(value.facts.removedProposalEvidence ?? []),
|
|
19
21
|
},
|
|
20
22
|
};
|
|
21
23
|
}
|
package/dist/src/index.d.ts
CHANGED
|
@@ -6,8 +6,10 @@ export type { ChangedResult, CheckResult, CheckResultCommon, ErrorResult, Lookou
|
|
|
6
6
|
export { defaultProviderResolver } from "./provider-resolution.js";
|
|
7
7
|
export type { ProviderResolver } from "./provider-resolution.js";
|
|
8
8
|
export { loadRegistry, LookoutRegistry, parseRegistry, RegistryValidationError, } from "./registry.js";
|
|
9
|
-
export type { LookoutRegistryDocument, LookoutSource, LookoutSourceKind, RenderPolicy, } from "./registry.js";
|
|
10
|
-
export { createLookoutSnapshotStore } from "./snapshot-store.js";
|
|
9
|
+
export type { ExtractableLookoutSource, LookoutRegistryDocument, LookoutSource, LookoutSourceKind, RenderPolicy, StructuredFileFormat, StructuredFileLookoutSource, } from "./registry.js";
|
|
10
|
+
export { createLookoutSnapshotStore, resolveLookoutSnapshot, } from "./snapshot-store.js";
|
|
11
|
+
export type { ResolveLookoutSnapshotOptions } from "./snapshot-store.js";
|
|
12
|
+
export type { SnapshotSourceRefResolution } from "@kontourai/forage/fetch";
|
|
11
13
|
export { canonicalValueKey } from "./canonical-value.js";
|
|
12
14
|
export type { CanonicalValueKey, CanonicalValueFormatVersion, CanonicalValueResult, DiffKernelError, DiffKernelErrorKind, DiffResult, IdentityResult, } from "./canonical-value.js";
|
|
13
15
|
export { compareStructural, diffKeyedMultiset } from "./structural-diff.js";
|
|
@@ -20,3 +22,7 @@ export { createDriftEmitter } from "./drift-emission.js";
|
|
|
20
22
|
export type { BaselineEstablishedFact, CreateDriftEmitterOptions, DriftEmitter, DriftError, DriftErrorKind, DriftFact, DriftResult, DriftSuccess, EmitDriftInput } from "./drift-emission.js";
|
|
21
23
|
export { checkSchemaCoverage } from "./coverage.js";
|
|
22
24
|
export type { SchemaCoverageGap, SchemaCoverageResult } from "./coverage.js";
|
|
25
|
+
export { createObserveExtractDiff } from "./observe-extract-diff.js";
|
|
26
|
+
export type { ObserveExtractAcquisition, ObserveExtractAttempt, ObserveExtractDiff, ObserveExtractDiffOptions, ObserveExtractError, ObserveExtractErrorKind, ObserveExtractExtraction, ObserveExtractExtractionInput, ObserveExtractObservation, ObserveExtractObservationIdentity, ObserveExtractOutcome, ObserveExtractProviderFailure, ObserveExtractRecorder, ObserveExtractResult, ObserveExtractSource, ObserveExtractSourceSnapshot, } from "./observe-extract-diff.js";
|
|
27
|
+
export { buildSemanticReviewWork, semanticReviewApiVersion } from "./semantic-review-work.js";
|
|
28
|
+
export type { BuildSemanticReviewWorkInput, SemanticClaimTarget, SemanticObservationIdentity, SemanticReviewCandidate, SemanticReviewChange, SemanticReviewItem, SemanticReviewKind, SemanticReviewWork, } from "./semantic-review-work.js";
|
package/dist/src/index.js
CHANGED
|
@@ -2,10 +2,12 @@ export { runCli } from "./cli.js";
|
|
|
2
2
|
export { createCheckRunner } from "./check-runner.js";
|
|
3
3
|
export { defaultProviderResolver } from "./provider-resolution.js";
|
|
4
4
|
export { loadRegistry, LookoutRegistry, parseRegistry, RegistryValidationError, } from "./registry.js";
|
|
5
|
-
export { createLookoutSnapshotStore } from "./snapshot-store.js";
|
|
5
|
+
export { createLookoutSnapshotStore, resolveLookoutSnapshot, } from "./snapshot-store.js";
|
|
6
6
|
export { canonicalValueKey } from "./canonical-value.js";
|
|
7
7
|
export { compareStructural, diffKeyedMultiset } from "./structural-diff.js";
|
|
8
8
|
export { diffProposalSets, extractionProposalIdentity } from "./proposal-diff.js";
|
|
9
9
|
export { createObservationStore } from "./observation-store.js";
|
|
10
10
|
export { createDriftEmitter } from "./drift-emission.js";
|
|
11
11
|
export { checkSchemaCoverage } from "./coverage.js";
|
|
12
|
+
export { createObserveExtractDiff } from "./observe-extract-diff.js";
|
|
13
|
+
export { buildSemanticReviewWork, semanticReviewApiVersion } from "./semantic-review-work.js";
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
import type { ExtractionPartial, ExtractionProviderFailure, ExtractionResult, PreparedArtifact } from "@kontourai/traverse";
|
|
2
|
+
import type { CheckResult } from "./check-result.js";
|
|
3
|
+
import type { ProposalSetObservation } from "./proposal-diff.js";
|
|
4
|
+
import type { LookoutSource } from "./registry.js";
|
|
5
|
+
/** Acquisition is supplied by the caller; Lookout does not add another fetcher. */
|
|
6
|
+
export interface ObserveExtractAcquisition {
|
|
7
|
+
check(source: LookoutSource): Promise<CheckResult>;
|
|
8
|
+
}
|
|
9
|
+
/**
|
|
10
|
+
* Extraction is supplied by the caller. The input carries only immutable source
|
|
11
|
+
* identity, leaving snapshot resolution, preparation, and provider selection
|
|
12
|
+
* outside Lookout.
|
|
13
|
+
*/
|
|
14
|
+
export interface ObserveExtractExtraction {
|
|
15
|
+
extract(input: ObserveExtractExtractionInput): Promise<ExtractionResult>;
|
|
16
|
+
}
|
|
17
|
+
export interface ObserveExtractExtractionInput {
|
|
18
|
+
readonly source: LookoutSource;
|
|
19
|
+
readonly snapshotRef: string;
|
|
20
|
+
}
|
|
21
|
+
export interface ObserveExtractAttempt {
|
|
22
|
+
readonly extractedAt: string;
|
|
23
|
+
readonly providerCalls: number;
|
|
24
|
+
readonly totalTokensUsed: number;
|
|
25
|
+
readonly partial?: ExtractionPartial;
|
|
26
|
+
readonly providerFailures?: readonly ObserveExtractProviderFailure[];
|
|
27
|
+
}
|
|
28
|
+
/** Provider-neutral durable classification; diagnostic payloads stay at the capability boundary. */
|
|
29
|
+
export interface ObserveExtractProviderFailure {
|
|
30
|
+
readonly kind: ExtractionProviderFailure["kind"];
|
|
31
|
+
readonly retryable: boolean;
|
|
32
|
+
}
|
|
33
|
+
export interface ObserveExtractSourceSnapshot {
|
|
34
|
+
readonly priorSnapshotRef: string | null;
|
|
35
|
+
readonly currentSnapshotRef: string;
|
|
36
|
+
}
|
|
37
|
+
export interface ObserveExtractSource {
|
|
38
|
+
readonly id: string;
|
|
39
|
+
readonly url: string;
|
|
40
|
+
readonly kind: LookoutSource["kind"];
|
|
41
|
+
}
|
|
42
|
+
export type ObserveExtractOutcome = "acquisition-error" | "unchanged" | "completed" | "partial" | "partial-provider-failure" | "provider-failure" | "extraction-failure";
|
|
43
|
+
/**
|
|
44
|
+
* One source observation, ready for caller-owned durable recording. Raw
|
|
45
|
+
* provider responses and source bodies are intentionally not copied here.
|
|
46
|
+
*/
|
|
47
|
+
export interface ObserveExtractObservation {
|
|
48
|
+
readonly source: ObserveExtractSource;
|
|
49
|
+
readonly check: CheckResult;
|
|
50
|
+
readonly outcome: ObserveExtractOutcome;
|
|
51
|
+
readonly sourceSnapshot: ObserveExtractSourceSnapshot | null;
|
|
52
|
+
readonly preparedArtifact: PreparedArtifact | null;
|
|
53
|
+
readonly proposalSet: ProposalSetObservation | null;
|
|
54
|
+
readonly attempt: ObserveExtractAttempt | null;
|
|
55
|
+
}
|
|
56
|
+
export interface ObserveExtractObservationIdentity {
|
|
57
|
+
readonly observationId: string;
|
|
58
|
+
readonly priorObservationId: string | null;
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* Lookout passes each completed observation to a caller-owned recorder. The
|
|
62
|
+
* recorder controls durable storage and continuity while this composition never
|
|
63
|
+
* supplies or configures the injected acquisition or extraction capabilities.
|
|
64
|
+
*/
|
|
65
|
+
export interface ObserveExtractRecorder {
|
|
66
|
+
record(observation: ObserveExtractObservation): Promise<ObserveExtractObservationIdentity>;
|
|
67
|
+
}
|
|
68
|
+
export interface ObserveExtractDiffOptions {
|
|
69
|
+
readonly acquisition: ObserveExtractAcquisition;
|
|
70
|
+
readonly extraction: ObserveExtractExtraction;
|
|
71
|
+
readonly recorder: ObserveExtractRecorder;
|
|
72
|
+
}
|
|
73
|
+
export type ObserveExtractErrorKind = "acquisition-threw" | "recording-failed" | "dependency-contract";
|
|
74
|
+
export interface ObserveExtractError {
|
|
75
|
+
readonly kind: ObserveExtractErrorKind;
|
|
76
|
+
readonly message: string;
|
|
77
|
+
readonly cause?: unknown;
|
|
78
|
+
}
|
|
79
|
+
export type ObserveExtractResult = {
|
|
80
|
+
readonly ok: true;
|
|
81
|
+
readonly value: ObserveExtractObservation & ObserveExtractObservationIdentity;
|
|
82
|
+
} | {
|
|
83
|
+
readonly ok: false;
|
|
84
|
+
readonly error: ObserveExtractError;
|
|
85
|
+
readonly observation?: ObserveExtractObservation;
|
|
86
|
+
};
|
|
87
|
+
export interface ObserveExtractDiff {
|
|
88
|
+
observe(source: LookoutSource): Promise<ObserveExtractResult>;
|
|
89
|
+
}
|
|
90
|
+
export declare function createObserveExtractDiff(options: ObserveExtractDiffOptions): ObserveExtractDiff;
|