@aigne/afs-index 1.12.0-beta.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE.md +26 -0
- package/dist/_virtual/_@oxc-project_runtime@0.108.0/helpers/decorate.cjs +11 -0
- package/dist/_virtual/_@oxc-project_runtime@0.108.0/helpers/decorate.mjs +10 -0
- package/dist/backend/d1-index-backend.cjs +656 -0
- package/dist/backend/d1-index-backend.d.cts +149 -0
- package/dist/backend/d1-index-backend.d.cts.map +1 -0
- package/dist/backend/d1-index-backend.d.mts +149 -0
- package/dist/backend/d1-index-backend.d.mts.map +1 -0
- package/dist/backend/d1-index-backend.mjs +657 -0
- package/dist/backend/d1-index-backend.mjs.map +1 -0
- package/dist/backend/d1-vector-backend.cjs +83 -0
- package/dist/backend/d1-vector-backend.d.cts +21 -0
- package/dist/backend/d1-vector-backend.d.cts.map +1 -0
- package/dist/backend/d1-vector-backend.d.mts +21 -0
- package/dist/backend/d1-vector-backend.d.mts.map +1 -0
- package/dist/backend/d1-vector-backend.mjs +84 -0
- package/dist/backend/d1-vector-backend.mjs.map +1 -0
- package/dist/backend/memory-backend.cjs +304 -0
- package/dist/backend/memory-backend.d.cts +66 -0
- package/dist/backend/memory-backend.d.cts.map +1 -0
- package/dist/backend/memory-backend.d.mts +66 -0
- package/dist/backend/memory-backend.d.mts.map +1 -0
- package/dist/backend/memory-backend.mjs +305 -0
- package/dist/backend/memory-backend.mjs.map +1 -0
- package/dist/backend/prefix-range.cjs +25 -0
- package/dist/backend/prefix-range.mjs +25 -0
- package/dist/backend/prefix-range.mjs.map +1 -0
- package/dist/backend/schema.cjs +187 -0
- package/dist/backend/schema.mjs +188 -0
- package/dist/backend/schema.mjs.map +1 -0
- package/dist/backend/sqlite-backend.cjs +889 -0
- package/dist/backend/sqlite-backend.d.cts +96 -0
- package/dist/backend/sqlite-backend.d.cts.map +1 -0
- package/dist/backend/sqlite-backend.d.mts +96 -0
- package/dist/backend/sqlite-backend.d.mts.map +1 -0
- package/dist/backend/sqlite-backend.mjs +890 -0
- package/dist/backend/sqlite-backend.mjs.map +1 -0
- package/dist/backend/types.d.cts +209 -0
- package/dist/backend/types.d.cts.map +1 -0
- package/dist/backend/types.d.mts +209 -0
- package/dist/backend/types.d.mts.map +1 -0
- package/dist/backend/upstash-vector-backend.cjs +98 -0
- package/dist/backend/upstash-vector-backend.d.cts +21 -0
- package/dist/backend/upstash-vector-backend.d.cts.map +1 -0
- package/dist/backend/upstash-vector-backend.d.mts +21 -0
- package/dist/backend/upstash-vector-backend.d.mts.map +1 -0
- package/dist/backend/upstash-vector-backend.mjs +98 -0
- package/dist/backend/upstash-vector-backend.mjs.map +1 -0
- package/dist/backend/vector-backend.d.cts +29 -0
- package/dist/backend/vector-backend.d.cts.map +1 -0
- package/dist/backend/vector-backend.d.mts +29 -0
- package/dist/backend/vector-backend.d.mts.map +1 -0
- package/dist/dirty-queue.d.cts +41 -0
- package/dist/dirty-queue.d.cts.map +1 -0
- package/dist/dirty-queue.d.mts +41 -0
- package/dist/dirty-queue.d.mts.map +1 -0
- package/dist/embed.cjs +45 -0
- package/dist/embed.d.cts +20 -0
- package/dist/embed.d.cts.map +1 -0
- package/dist/embed.d.mts +20 -0
- package/dist/embed.d.mts.map +1 -0
- package/dist/embed.mjs +46 -0
- package/dist/embed.mjs.map +1 -0
- package/dist/extraction/defaults.cjs +242 -0
- package/dist/extraction/defaults.d.cts +14 -0
- package/dist/extraction/defaults.d.cts.map +1 -0
- package/dist/extraction/defaults.d.mts +14 -0
- package/dist/extraction/defaults.d.mts.map +1 -0
- package/dist/extraction/defaults.mjs +242 -0
- package/dist/extraction/defaults.mjs.map +1 -0
- package/dist/extraction/field-map.cjs +162 -0
- package/dist/extraction/field-map.d.cts +70 -0
- package/dist/extraction/field-map.d.cts.map +1 -0
- package/dist/extraction/field-map.d.mts +70 -0
- package/dist/extraction/field-map.d.mts.map +1 -0
- package/dist/extraction/field-map.mjs +157 -0
- package/dist/extraction/field-map.mjs.map +1 -0
- package/dist/extraction/manager.cjs +74 -0
- package/dist/extraction/manager.d.cts +47 -0
- package/dist/extraction/manager.d.cts.map +1 -0
- package/dist/extraction/manager.d.mts +47 -0
- package/dist/extraction/manager.d.mts.map +1 -0
- package/dist/extraction/manager.mjs +74 -0
- package/dist/extraction/manager.mjs.map +1 -0
- package/dist/extraction/path-anchors.cjs +86 -0
- package/dist/extraction/path-anchors.d.cts +72 -0
- package/dist/extraction/path-anchors.d.cts.map +1 -0
- package/dist/extraction/path-anchors.d.mts +72 -0
- package/dist/extraction/path-anchors.d.mts.map +1 -0
- package/dist/extraction/path-anchors.mjs +86 -0
- package/dist/extraction/path-anchors.mjs.map +1 -0
- package/dist/follower-scheduler.cjs +92 -0
- package/dist/follower-scheduler.d.cts +58 -0
- package/dist/follower-scheduler.d.cts.map +1 -0
- package/dist/follower-scheduler.d.mts +58 -0
- package/dist/follower-scheduler.d.mts.map +1 -0
- package/dist/follower-scheduler.mjs +92 -0
- package/dist/follower-scheduler.mjs.map +1 -0
- package/dist/follower.cjs +577 -0
- package/dist/follower.d.cts +241 -0
- package/dist/follower.d.cts.map +1 -0
- package/dist/follower.d.mts +241 -0
- package/dist/follower.d.mts.map +1 -0
- package/dist/follower.mjs +571 -0
- package/dist/follower.mjs.map +1 -0
- package/dist/index-provider.cjs +1166 -0
- package/dist/index-provider.d.cts +175 -0
- package/dist/index-provider.d.cts.map +1 -0
- package/dist/index-provider.d.mts +175 -0
- package/dist/index-provider.d.mts.map +1 -0
- package/dist/index-provider.mjs +1167 -0
- package/dist/index-provider.mjs.map +1 -0
- package/dist/index.cjs +53 -0
- package/dist/index.d.cts +22 -0
- package/dist/index.d.mts +22 -0
- package/dist/index.mjs +20 -0
- package/dist/keys/exact.cjs +20 -0
- package/dist/keys/exact.mjs +20 -0
- package/dist/keys/exact.mjs.map +1 -0
- package/dist/keys/location.cjs +13 -0
- package/dist/keys/location.mjs +13 -0
- package/dist/keys/location.mjs.map +1 -0
- package/dist/keys/normalized.cjs +62 -0
- package/dist/keys/normalized.mjs +63 -0
- package/dist/keys/normalized.mjs.map +1 -0
- package/dist/keys/person.cjs +15 -0
- package/dist/keys/person.mjs +15 -0
- package/dist/keys/person.mjs.map +1 -0
- package/dist/keys/registry.cjs +45 -0
- package/dist/keys/registry.d.cts +15 -0
- package/dist/keys/registry.d.cts.map +1 -0
- package/dist/keys/registry.d.mts +15 -0
- package/dist/keys/registry.d.mts.map +1 -0
- package/dist/keys/registry.mjs +46 -0
- package/dist/keys/registry.mjs.map +1 -0
- package/dist/keys/time.cjs +128 -0
- package/dist/keys/time.mjs +128 -0
- package/dist/keys/time.mjs.map +1 -0
- package/dist/keys/topic.cjs +23 -0
- package/dist/keys/topic.mjs +23 -0
- package/dist/keys/topic.mjs.map +1 -0
- package/dist/manifest-index.cjs +29 -0
- package/dist/manifest-index.d.cts +24 -0
- package/dist/manifest-index.d.cts.map +1 -0
- package/dist/manifest-index.d.mts +24 -0
- package/dist/manifest-index.d.mts.map +1 -0
- package/dist/manifest-index.mjs +28 -0
- package/dist/manifest-index.mjs.map +1 -0
- package/dist/offline-rebuild.cjs +48 -0
- package/dist/offline-rebuild.d.cts +57 -0
- package/dist/offline-rebuild.d.cts.map +1 -0
- package/dist/offline-rebuild.d.mts +57 -0
- package/dist/offline-rebuild.d.mts.map +1 -0
- package/dist/offline-rebuild.mjs +49 -0
- package/dist/offline-rebuild.mjs.map +1 -0
- package/dist/query/engine.cjs +433 -0
- package/dist/query/engine.d.cts +34 -0
- package/dist/query/engine.d.cts.map +1 -0
- package/dist/query/engine.d.mts +34 -0
- package/dist/query/engine.d.mts.map +1 -0
- package/dist/query/engine.mjs +433 -0
- package/dist/query/engine.mjs.map +1 -0
- package/dist/query/limits.cjs +12 -0
- package/dist/query/limits.mjs +12 -0
- package/dist/query/limits.mjs.map +1 -0
- package/dist/text/cjk.cjs +30 -0
- package/dist/text/cjk.d.cts +24 -0
- package/dist/text/cjk.d.cts.map +1 -0
- package/dist/text/cjk.d.mts +24 -0
- package/dist/text/cjk.d.mts.map +1 -0
- package/dist/text/cjk.mjs +30 -0
- package/dist/text/cjk.mjs.map +1 -0
- package/dist/text/fts.cjs +244 -0
- package/dist/text/fts.mjs +245 -0
- package/dist/text/fts.mjs.map +1 -0
- package/dist/text/host.cjs +39 -0
- package/dist/text/host.mjs +38 -0
- package/dist/text/host.mjs.map +1 -0
- package/dist/types.cjs +44 -0
- package/dist/types.d.cts +393 -0
- package/dist/types.d.cts.map +1 -0
- package/dist/types.d.mts +393 -0
- package/dist/types.d.mts.map +1 -0
- package/dist/types.mjs +42 -0
- package/dist/types.mjs.map +1 -0
- package/package.json +59 -0
|
@@ -0,0 +1,571 @@
|
|
|
1
|
+
import { applyEnrichContentFile, applyFieldMap, extractSummaryText } from "./extraction/field-map.mjs";
|
|
2
|
+
import { AFSNotFoundError, makeNsLog } from "@aigne/afs";
|
|
3
|
+
import { joinURL } from "ufo";
|
|
4
|
+
|
|
5
|
+
//#region src/follower.ts
|
|
6
|
+
/**
|
|
7
|
+
* IndexFollower — DRAINS the source-side dirty queue (`ds_dirty_path`, the merged
|
|
8
|
+
* current-state projection of the changelog) and incrementally indexes file
|
|
9
|
+
* writes/deletes into the index backend. Dirty drain is the SOLE mechanism —
|
|
10
|
+
* the changelog `/.changelog` replay was deleted at the §13 cutover; a follower
|
|
11
|
+
* with no `dirtyQueue` is inert.
|
|
12
|
+
*
|
|
13
|
+
* Two entry points:
|
|
14
|
+
* runCatchUp() — inline drain; called from execQuery so a query always sees a
|
|
15
|
+
* fresh index (processes whatever dirty rows are pending).
|
|
16
|
+
* runBatch() — bounded per-tick drain (maxProcessed/maxNoop/maxScan caps);
|
|
17
|
+
* called from the cron task.
|
|
18
|
+
*
|
|
19
|
+
* Cursor: a `consumer_id` row in `idx_index_cursor` registers the follower and
|
|
20
|
+
* carries its instanceDid + domains snapshot for durable cron discovery. Its
|
|
21
|
+
* `last_seq` is a DEAD column post-cutover (nothing advances it during drain —
|
|
22
|
+
* §4.2); scheduling fairness rides `served_at` (markCursorServed). All domains a
|
|
23
|
+
* SINGLE blocklet declares share one cursor; DIFFERENT blocklets under the same
|
|
24
|
+
* caller each get their own (their `/user` data lives in separate instance
|
|
25
|
+
* spaces / dirty-queue prefixes).
|
|
26
|
+
*
|
|
27
|
+
* Consumer-id identity (design.md §4.4 decision-lock addendum, owner
|
|
28
|
+
* 2026-07-05; per-instance fix): a single shared backend hosts MULTIPLE
|
|
29
|
+
* `(scope, instanceDid)` pairs at once — Node's one-shared-SQLite-file rework
|
|
30
|
+
* of #936 P4, and CF's one-shared-D1 model. So the bare `FOLLOWER_CONSUMER_ID`
|
|
31
|
+
* isn't enough to identify ONE follower's cursor row;
|
|
32
|
+
* `deriveFollowerConsumerId(scope, instanceDid)` folds BOTH in
|
|
33
|
+
* (`index:{scope}:{instanceDid}`), the same compositing technique already used
|
|
34
|
+
* for domain-name isolation (`{declared name}:{scope}`). Keying by scope ALONE
|
|
35
|
+
* (the pre-fix behavior) let a caller's SECOND index-blocklet collide on the
|
|
36
|
+
* first's cursor row (`INSERT OR IGNORE` → dropped) so its changelog was never
|
|
37
|
+
* followed. D1 additionally isolates via its own `scope` column (PRIMARY KEY
|
|
38
|
+
* (scope, consumer_id)); both runtimes derive the consumer_id the same way so a
|
|
39
|
+
* persisted row means the same thing everywhere (`IndexBackend.listCursorScopes`
|
|
40
|
+
* matches on the `{FOLLOWER_CONSUMER_ID}:` PREFIX for exactly this reason — see
|
|
41
|
+
* d1-index-backend.ts).
|
|
42
|
+
*/
|
|
43
|
+
const log = makeNsLog("provider:index");
|
|
44
|
+
const FOLLOWER_CONSUMER_ID = "index";
|
|
45
|
+
/**
|
|
46
|
+
* Derive a follower consumer_id that isolates per `(scope, instanceDid)` —
|
|
47
|
+
* `"index:{scope}:{instanceDid}"`. A single shared backend (Node's `/user/index`
|
|
48
|
+
* file, CF's D1) hosts many callers AND many blocklets per caller; the cursor
|
|
49
|
+
* row must be unique per (caller scope × blocklet instance) because each
|
|
50
|
+
* blocklet follows its OWN instance-space changelog. `instanceDid` is optional
|
|
51
|
+
* for backward compatibility (bare `index:{scope}` for pre-fix rows / callers
|
|
52
|
+
* that don't supply it); passing the bare `FOLLOWER_CONSUMER_ID` is only safe
|
|
53
|
+
* for single-scope callers (tests, standalone use).
|
|
54
|
+
*/
|
|
55
|
+
function deriveFollowerConsumerId(scope, instanceDid) {
|
|
56
|
+
return instanceDid ? `${FOLLOWER_CONSUMER_ID}:${scope}:${instanceDid}` : `${FOLLOWER_CONSUMER_ID}:${scope}`;
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
59
|
+
* Memory-map key for a caller's per-blocklet index resources (the AFSIndex
|
|
60
|
+
* handle, its in-flight build promise, its IndexFollower). Isolation is
|
|
61
|
+
* per-`(scope, instanceDid)`: the SAME caller (scope) gets a SEPARATE index per
|
|
62
|
+
* blocklet (instanceDid). Keying these isolate-lived maps by scope ALONE let a
|
|
63
|
+
* caller's second index-blocklet reuse the FIRST's handle/follower (wrong bound
|
|
64
|
+
* domains + wrong instance-space changelog) — the in-memory twin of the cursor
|
|
65
|
+
* collision `deriveFollowerConsumerId` guards against.
|
|
66
|
+
*/
|
|
67
|
+
function userIndexKey(scope, instanceDid) {
|
|
68
|
+
return instanceDid ? `${scope}::${instanceDid}` : scope;
|
|
69
|
+
}
|
|
70
|
+
const DEFAULT_MAX_PROCESSED_PER_TICK = 4;
|
|
71
|
+
const DEFAULT_MAX_NOOP_PER_TICK = 12;
|
|
72
|
+
const DEFAULT_MAX_SCAN_PER_TICK = 400;
|
|
73
|
+
/** Preferred alias for {@link DEFAULT_MAX_SCAN_PER_TICK} in dirty-drain call sites. */
|
|
74
|
+
const DEFAULT_MAX_DIRTY_ROWS_PER_TICK = DEFAULT_MAX_SCAN_PER_TICK;
|
|
75
|
+
const DIRTY_RETRY_BACKOFF_MS = 6e4;
|
|
76
|
+
function toAbsolute(path) {
|
|
77
|
+
return joinURL("/", path);
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* Internal signal thrown by `indexEntry` / the dirty-drain remove path when a
|
|
81
|
+
* read/stat/index failed on content that is STILL current (issue #935 BLOCK
|
|
82
|
+
* fix #3 / sync-engine parity): `processDirty` catches this and BACKS OFF the
|
|
83
|
+
* dirty row (marks it, doesn't ack) so the next tick retries it instead of
|
|
84
|
+
* silently losing the write. (Historically named for the changelog replay that
|
|
85
|
+
* the dirty-path cutover deleted; kept as the dirty-drain retry signal.)
|
|
86
|
+
*/
|
|
87
|
+
var ChangelogAbortError = class extends Error {
|
|
88
|
+
constructor(originalPath, reason = "transient read failure") {
|
|
89
|
+
super(`${reason} for "${originalPath}" — retry next run`);
|
|
90
|
+
this.name = "ChangelogAbortError";
|
|
91
|
+
}
|
|
92
|
+
};
|
|
93
|
+
var IndexFollower = class {
|
|
94
|
+
spaceAfs;
|
|
95
|
+
backend;
|
|
96
|
+
domains;
|
|
97
|
+
consumerId;
|
|
98
|
+
embed;
|
|
99
|
+
catchUpLimits;
|
|
100
|
+
dirtyQueue;
|
|
101
|
+
constructor(opts) {
|
|
102
|
+
this.spaceAfs = opts.spaceAfs;
|
|
103
|
+
this.backend = opts.backend;
|
|
104
|
+
this.domains = opts.domains;
|
|
105
|
+
this.consumerId = opts.consumerId ?? FOLLOWER_CONSUMER_ID;
|
|
106
|
+
this.embed = opts.embed;
|
|
107
|
+
this.dirtyQueue = opts.dirtyQueue;
|
|
108
|
+
}
|
|
109
|
+
/**
|
|
110
|
+
* Override the inline `runCatchUp()` budget — the per-invocation caps on
|
|
111
|
+
* matched PUT attempts, matched delete/remove rows, and dirty rows FETCHED
|
|
112
|
+
* (`maxScan`). Injected by the runtime from the SAME env group the cron uses
|
|
113
|
+
* (`AFS_INDEX_MAX_*_PER_TICK`) so both indexing paths share one budget group.
|
|
114
|
+
* Unset → provider defaults (standalone/tests).
|
|
115
|
+
*/
|
|
116
|
+
setCatchUpLimits(limits) {
|
|
117
|
+
this.catchUpLimits = limits;
|
|
118
|
+
}
|
|
119
|
+
/**
|
|
120
|
+
* Wire (or replace) the passage embedder used at index time — semantic recall
|
|
121
|
+
* (Epic #932). Called by `AFSIndex.setFollower` (query-path follower) and by
|
|
122
|
+
* `rebuildFollowerFromCursor` (CF cron-path follower) so BOTH indexing paths
|
|
123
|
+
* write embeddings; without this a changelog-indexed entry has only FTS +
|
|
124
|
+
* anchors and cross-lingual / semantic queries can't match it. A no-op
|
|
125
|
+
* embedder (or a backend without `writeEmbedding`) degrades cleanly to
|
|
126
|
+
* FTS+anchor.
|
|
127
|
+
*/
|
|
128
|
+
setEmbedder(embed) {
|
|
129
|
+
this.embed = embed;
|
|
130
|
+
}
|
|
131
|
+
/**
|
|
132
|
+
* Lightweight catch-up: drain any ready dirty rows so a query sees a fresh
|
|
133
|
+
* index without waiting for the next cron tick. Called from execQuery.
|
|
134
|
+
*
|
|
135
|
+
* Dirty drain is the SOLE catch-up mechanism (index-follower-dirty-path §13
|
|
136
|
+
* cutover — the changelog `/.changelog` replay was deleted). Freshness = "are
|
|
137
|
+
* there ready dirty rows under my prefix", so there is no `last_seq` probe: a
|
|
138
|
+
* query sees a fresh index by processing whatever dirty paths are pending
|
|
139
|
+
* (design §4.2, §8). A follower with no dirty queue (standalone/legacy) is a
|
|
140
|
+
* no-op here — it has no source to drain.
|
|
141
|
+
*/
|
|
142
|
+
async runCatchUp() {
|
|
143
|
+
if (!this.dirtyQueue) return;
|
|
144
|
+
await this.processDirty({
|
|
145
|
+
maxProcessed: this.catchUpLimits?.maxProcessed ?? DEFAULT_MAX_PROCESSED_PER_TICK,
|
|
146
|
+
maxNoop: this.catchUpLimits?.maxNoop ?? DEFAULT_MAX_NOOP_PER_TICK,
|
|
147
|
+
maxDirtyRows: this.catchUpLimits?.maxScan ?? DEFAULT_MAX_DIRTY_ROWS_PER_TICK
|
|
148
|
+
});
|
|
149
|
+
}
|
|
150
|
+
/**
|
|
151
|
+
* Bounded per-tick drain of the dirty queue — attempt up to `maxProcessed`
|
|
152
|
+
* matched PUT rows, up to `maxNoop` matched delete/remove/noop rows, fetching
|
|
153
|
+
* at most `maxScan` (a.k.a. `maxDirtyRows`) dirty rows. Called from the cron.
|
|
154
|
+
*
|
|
155
|
+
* Dirty drain is the SOLE mechanism (index-follower-dirty-path §13 cutover —
|
|
156
|
+
* changelog replay was deleted). A follower with no dirty queue has no source
|
|
157
|
+
* to drain, so it is idle. Separate budgets are intentional: matched PUT rows
|
|
158
|
+
* are the expensive read+embed+write axis, delete/noop rows are cheaper.
|
|
159
|
+
*/
|
|
160
|
+
async runBatch(limits) {
|
|
161
|
+
if (!this.dirtyQueue) return this.idleResult();
|
|
162
|
+
return this.processDirty({
|
|
163
|
+
maxProcessed: limits?.maxProcessed ?? DEFAULT_MAX_PROCESSED_PER_TICK,
|
|
164
|
+
maxNoop: limits?.maxNoop ?? DEFAULT_MAX_NOOP_PER_TICK,
|
|
165
|
+
maxDirtyRows: limits?.maxDirtyRows ?? limits?.maxScan ?? DEFAULT_MAX_DIRTY_ROWS_PER_TICK
|
|
166
|
+
});
|
|
167
|
+
}
|
|
168
|
+
/** Empty batch result (no dirty queue wired / nothing to drain). */
|
|
169
|
+
idleResult() {
|
|
170
|
+
return {
|
|
171
|
+
scanned: 0,
|
|
172
|
+
processed: 0,
|
|
173
|
+
noopProcessed: 0,
|
|
174
|
+
indexed: 0,
|
|
175
|
+
deleted: 0,
|
|
176
|
+
nooped: 0,
|
|
177
|
+
advanced: false,
|
|
178
|
+
hasMore: false,
|
|
179
|
+
ackRace: false,
|
|
180
|
+
stopReason: "exhausted",
|
|
181
|
+
fromSeq: 0,
|
|
182
|
+
toSeq: 0,
|
|
183
|
+
matched: 0,
|
|
184
|
+
unmatched: 0,
|
|
185
|
+
skipped: 0,
|
|
186
|
+
failed: 0,
|
|
187
|
+
aborted: false
|
|
188
|
+
};
|
|
189
|
+
}
|
|
190
|
+
/**
|
|
191
|
+
* Cold rebuild of an EXPLICIT list of space-relative paths — the enumeration
|
|
192
|
+
* is done by the caller (an external driver listing the live tree via D1),
|
|
193
|
+
* NOT in this process. Reads CURRENT live state per path and writes the
|
|
194
|
+
* derived index layers, independent of any queue — so it is the seam the
|
|
195
|
+
* offline reset-and-rebuild (`offlineRebuildScope`) uses to rebuild from live
|
|
196
|
+
* `ds_entries`. Keeping the per-call cost to just `paths.length`
|
|
197
|
+
* read+embed+writes with ZERO in-Worker tree enumeration is what keeps a small
|
|
198
|
+
* batch inside the CF free-tier per-request CPU/subrequest budget (an in-Worker
|
|
199
|
+
* `maxDepth:-1` enumeration pushed cold isolates over that budget into 503s).
|
|
200
|
+
* Each path is matched against the declared domains and indexed through the
|
|
201
|
+
* same `indexEntry` as the drain path (byte-identical output). `paths` are
|
|
202
|
+
* space-relative, e.g. `"items/foo.json"` (leading slash tolerated).
|
|
203
|
+
*/
|
|
204
|
+
async reindexPaths(paths, onProgress) {
|
|
205
|
+
let indexed = 0;
|
|
206
|
+
let failed = 0;
|
|
207
|
+
let skipped = 0;
|
|
208
|
+
let done = 0;
|
|
209
|
+
for (const raw of paths) {
|
|
210
|
+
const rel = raw.replace(/^\/+/, "");
|
|
211
|
+
let matched = false;
|
|
212
|
+
for (const domain of this.domains) {
|
|
213
|
+
if (!this.pathMatchesDomain(rel, domain)) continue;
|
|
214
|
+
matched = true;
|
|
215
|
+
try {
|
|
216
|
+
if (await this.indexEntry(domain, rel, this.makeEntryPath(domain, rel)) === "indexed") indexed++;
|
|
217
|
+
} catch (err) {
|
|
218
|
+
failed++;
|
|
219
|
+
log.warn(`reindex failed: ${rel}`, err instanceof Error ? err.message : err);
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
if (!matched) skipped++;
|
|
223
|
+
done++;
|
|
224
|
+
if (onProgress && (done % 50 === 0 || done === paths.length)) onProgress(done, paths.length);
|
|
225
|
+
}
|
|
226
|
+
return {
|
|
227
|
+
indexed,
|
|
228
|
+
failed,
|
|
229
|
+
skipped
|
|
230
|
+
};
|
|
231
|
+
}
|
|
232
|
+
/**
|
|
233
|
+
* Dirty drain (index-follower-dirty-path §8) — the O(unique dirty paths)
|
|
234
|
+
* replacement for changelog replay. Fetches ready dirty rows under this
|
|
235
|
+
* follower's prefix, and for each: domain-matches the stripped `originalPath`,
|
|
236
|
+
* reads CURRENT live state, and indexes / removes accordingly, then acks with
|
|
237
|
+
* the `latestSeq` race guard. A transient failure marks the row with a backoff
|
|
238
|
+
* (so a poison row can't refill the tick) instead of acking.
|
|
239
|
+
*
|
|
240
|
+
* Dirty mode NEVER writes `idx_index_cursor.last_seq` (design §4.2 — it is a
|
|
241
|
+
* dead column post-cutover); scheduling reads `advanced/hasMore/scanned` +
|
|
242
|
+
* the dirty count. The `dirtyQueue.listDirtyPaths({ prefix: "" })` call assumes
|
|
243
|
+
* a fragment-scoped queue (PrefixedTreeIndex), so `row.path` is already the
|
|
244
|
+
* stripped `originalPath` the domain globs + `spaceAfs.read` expect.
|
|
245
|
+
*/
|
|
246
|
+
async processDirty(limits) {
|
|
247
|
+
const result0 = () => ({
|
|
248
|
+
scanned: 0,
|
|
249
|
+
processed: 0,
|
|
250
|
+
noopProcessed: 0,
|
|
251
|
+
indexed: 0,
|
|
252
|
+
deleted: 0,
|
|
253
|
+
nooped: 0,
|
|
254
|
+
advanced: false,
|
|
255
|
+
hasMore: false,
|
|
256
|
+
ackRace: false,
|
|
257
|
+
stopReason: "exhausted",
|
|
258
|
+
fromSeq: 0,
|
|
259
|
+
toSeq: 0,
|
|
260
|
+
matched: 0,
|
|
261
|
+
unmatched: 0,
|
|
262
|
+
skipped: 0,
|
|
263
|
+
failed: 0,
|
|
264
|
+
aborted: false
|
|
265
|
+
});
|
|
266
|
+
const dirtyQueue = this.dirtyQueue;
|
|
267
|
+
if (!dirtyQueue) return result0();
|
|
268
|
+
const maxDirtyRows = limits.maxDirtyRows > 0 ? limits.maxDirtyRows : 0;
|
|
269
|
+
if (maxDirtyRows <= 0 || limits.maxProcessed <= 0 && limits.maxNoop <= 0) return result0();
|
|
270
|
+
let processedRemaining = limits.maxProcessed;
|
|
271
|
+
let noopRemaining = limits.maxNoop;
|
|
272
|
+
let scanned = 0;
|
|
273
|
+
let processed = 0;
|
|
274
|
+
let noopProcessed = 0;
|
|
275
|
+
let indexed = 0;
|
|
276
|
+
let deleted = 0;
|
|
277
|
+
let nooped = 0;
|
|
278
|
+
let matched = 0;
|
|
279
|
+
let unmatched = 0;
|
|
280
|
+
let failed = 0;
|
|
281
|
+
let advanced = false;
|
|
282
|
+
let hasMore = false;
|
|
283
|
+
let ackRace = false;
|
|
284
|
+
let stopReason;
|
|
285
|
+
const now = Date.now();
|
|
286
|
+
const rows = await dirtyQueue.listDirtyPaths({
|
|
287
|
+
prefix: "",
|
|
288
|
+
limit: maxDirtyRows,
|
|
289
|
+
now
|
|
290
|
+
});
|
|
291
|
+
for (const row of rows) {
|
|
292
|
+
scanned++;
|
|
293
|
+
const matchedDomains = this.domains.filter((d) => this.pathMatchesDomain(row.path, d));
|
|
294
|
+
if (matchedDomains.length === 0) {
|
|
295
|
+
if (noopRemaining <= 0) {
|
|
296
|
+
hasMore = true;
|
|
297
|
+
stopReason = "noop_budget";
|
|
298
|
+
break;
|
|
299
|
+
}
|
|
300
|
+
unmatched++;
|
|
301
|
+
noopRemaining--;
|
|
302
|
+
noopProcessed++;
|
|
303
|
+
if (await dirtyQueue.ackDirtyPath(row.path, row.latestSeq)) advanced = true;
|
|
304
|
+
else {
|
|
305
|
+
hasMore = true;
|
|
306
|
+
ackRace = true;
|
|
307
|
+
}
|
|
308
|
+
continue;
|
|
309
|
+
}
|
|
310
|
+
const isDelete = row.op === "delete";
|
|
311
|
+
if (isDelete) {
|
|
312
|
+
if (noopRemaining <= 0) {
|
|
313
|
+
hasMore = true;
|
|
314
|
+
stopReason = "noop_budget";
|
|
315
|
+
break;
|
|
316
|
+
}
|
|
317
|
+
} else if (processedRemaining <= 0) {
|
|
318
|
+
hasMore = true;
|
|
319
|
+
stopReason = "processed_budget";
|
|
320
|
+
break;
|
|
321
|
+
}
|
|
322
|
+
try {
|
|
323
|
+
let didIndex = 0;
|
|
324
|
+
let didRemove = 0;
|
|
325
|
+
if (isDelete) {
|
|
326
|
+
await this.removeDirty(matchedDomains, row.path);
|
|
327
|
+
didRemove = 1;
|
|
328
|
+
} else {
|
|
329
|
+
const r = await this.indexPutDirty(matchedDomains, row.path);
|
|
330
|
+
didIndex = r.indexed;
|
|
331
|
+
didRemove = r.removed;
|
|
332
|
+
}
|
|
333
|
+
matched++;
|
|
334
|
+
if (isDelete) {
|
|
335
|
+
noopRemaining--;
|
|
336
|
+
noopProcessed++;
|
|
337
|
+
deleted++;
|
|
338
|
+
} else if (didIndex > 0) {
|
|
339
|
+
processedRemaining--;
|
|
340
|
+
processed++;
|
|
341
|
+
indexed++;
|
|
342
|
+
} else if (didRemove > 0) {
|
|
343
|
+
noopProcessed++;
|
|
344
|
+
deleted++;
|
|
345
|
+
} else nooped++;
|
|
346
|
+
if (await dirtyQueue.ackDirtyPath(row.path, row.latestSeq)) advanced = true;
|
|
347
|
+
else {
|
|
348
|
+
advanced = true;
|
|
349
|
+
hasMore = true;
|
|
350
|
+
ackRace = true;
|
|
351
|
+
}
|
|
352
|
+
} catch (err) {
|
|
353
|
+
failed++;
|
|
354
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
355
|
+
await dirtyQueue.markDirtyPathError(row.path, row.latestSeq, msg, now + DIRTY_RETRY_BACKOFF_MS);
|
|
356
|
+
log.warn(`dirty drain failed (backed off): ${row.path}`, msg);
|
|
357
|
+
}
|
|
358
|
+
if (processedRemaining <= 0 && noopRemaining <= 0) {
|
|
359
|
+
hasMore = true;
|
|
360
|
+
stopReason = "processed_budget";
|
|
361
|
+
break;
|
|
362
|
+
}
|
|
363
|
+
}
|
|
364
|
+
if (!stopReason) {
|
|
365
|
+
if (rows.length >= maxDirtyRows && maxDirtyRows > 0) {
|
|
366
|
+
if ((await dirtyQueue.listDirtyPaths({
|
|
367
|
+
prefix: "",
|
|
368
|
+
limit: 1,
|
|
369
|
+
now
|
|
370
|
+
})).length > 0) hasMore = true;
|
|
371
|
+
}
|
|
372
|
+
stopReason = hasMore ? "scan_budget" : "exhausted";
|
|
373
|
+
}
|
|
374
|
+
log.debug("dirty drain processed", {
|
|
375
|
+
consumerId: this.consumerId,
|
|
376
|
+
scanned,
|
|
377
|
+
processed,
|
|
378
|
+
noopProcessed,
|
|
379
|
+
indexed,
|
|
380
|
+
deleted,
|
|
381
|
+
nooped,
|
|
382
|
+
matched,
|
|
383
|
+
unmatched,
|
|
384
|
+
failed,
|
|
385
|
+
advanced,
|
|
386
|
+
hasMore,
|
|
387
|
+
ackRace,
|
|
388
|
+
stopReason
|
|
389
|
+
});
|
|
390
|
+
return {
|
|
391
|
+
scanned,
|
|
392
|
+
processed,
|
|
393
|
+
noopProcessed,
|
|
394
|
+
indexed,
|
|
395
|
+
deleted,
|
|
396
|
+
nooped,
|
|
397
|
+
advanced,
|
|
398
|
+
hasMore,
|
|
399
|
+
ackRace,
|
|
400
|
+
stopReason,
|
|
401
|
+
fromSeq: 0,
|
|
402
|
+
toSeq: 0,
|
|
403
|
+
matched,
|
|
404
|
+
unmatched,
|
|
405
|
+
skipped: 0,
|
|
406
|
+
failed,
|
|
407
|
+
aborted: false
|
|
408
|
+
};
|
|
409
|
+
}
|
|
410
|
+
/** Remove the indexed doc(s) for a delete/removed dirty row across matched domains. */
|
|
411
|
+
async removeDirty(matchedDomains, originalPath) {
|
|
412
|
+
for (const domain of matchedDomains) await this.removeIndexed(this.makeEntryPath(domain, originalPath), originalPath);
|
|
413
|
+
}
|
|
414
|
+
/** Shared index-removal (delete branch) — throws ChangelogAbortError on failure so the row backs off. */
|
|
415
|
+
async removeIndexed(entryPath, originalPath) {
|
|
416
|
+
try {
|
|
417
|
+
if (this.backend.removeIndexedDocument) await this.backend.removeIndexedDocument(entryPath);
|
|
418
|
+
else {
|
|
419
|
+
await this.backend.removeAnchors(entryPath);
|
|
420
|
+
await this.backend.removeSummary(entryPath);
|
|
421
|
+
await this.backend.removeEntry(entryPath);
|
|
422
|
+
}
|
|
423
|
+
} catch (err) {
|
|
424
|
+
throw new ChangelogAbortError(originalPath, `index delete failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
425
|
+
}
|
|
426
|
+
}
|
|
427
|
+
/**
|
|
428
|
+
* Handle a `put` dirty row against CURRENT live state (design §4.3 / §8 step 6):
|
|
429
|
+
* - live exists → indexEntry (writes FTS/anchor/summary/embedding).
|
|
430
|
+
* - live missing (authoritative 404) → remove the indexed doc.
|
|
431
|
+
* - stat throws a NON-404 error → transient → ChangelogAbortError (retry).
|
|
432
|
+
* This is what makes a stale put whose file was deleted converge to "removed",
|
|
433
|
+
* while a replica-lag / D1 blip is NEVER mistaken for a deletion.
|
|
434
|
+
*/
|
|
435
|
+
async indexPutDirty(matchedDomains, originalPath) {
|
|
436
|
+
let live;
|
|
437
|
+
try {
|
|
438
|
+
live = (await this.spaceAfs.stat?.(toAbsolute(originalPath)))?.data;
|
|
439
|
+
} catch (err) {
|
|
440
|
+
if (err instanceof AFSNotFoundError) live = void 0;
|
|
441
|
+
else throw new ChangelogAbortError(originalPath, `stat failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
442
|
+
}
|
|
443
|
+
if (!live) {
|
|
444
|
+
await this.removeDirty(matchedDomains, originalPath);
|
|
445
|
+
return {
|
|
446
|
+
indexed: 0,
|
|
447
|
+
removed: 1
|
|
448
|
+
};
|
|
449
|
+
}
|
|
450
|
+
let indexed = 0;
|
|
451
|
+
for (const domain of matchedDomains) if (await this.indexEntry(domain, originalPath, this.makeEntryPath(domain, originalPath)) === "indexed") indexed++;
|
|
452
|
+
return {
|
|
453
|
+
indexed,
|
|
454
|
+
removed: 0
|
|
455
|
+
};
|
|
456
|
+
}
|
|
457
|
+
/**
|
|
458
|
+
* Does `originalPath` STILL EXIST (via `stat`)? Used to classify a `read()`
|
|
459
|
+
* failure (issue #935 BLOCK fix #3 / sync-engine parity):
|
|
460
|
+
* - stat 404 (or throws) → the file is gone (later delete, or never existed
|
|
461
|
+
* at this position) → the row is stale, safe to skip.
|
|
462
|
+
* - stat succeeds → the file is live; the read failure was transient (R2/D1
|
|
463
|
+
* blip) and must be retried, not lost.
|
|
464
|
+
* Post-cutover the drain only ever indexes live state (cid is never compared —
|
|
465
|
+
* supersession is caught by the ack race-guard), so existence is the only
|
|
466
|
+
* question left here.
|
|
467
|
+
*/
|
|
468
|
+
async entryStillCurrent(originalPath) {
|
|
469
|
+
try {
|
|
470
|
+
return !!(await this.spaceAfs.stat?.(toAbsolute(originalPath)))?.data;
|
|
471
|
+
} catch {
|
|
472
|
+
return false;
|
|
473
|
+
}
|
|
474
|
+
}
|
|
475
|
+
pathMatchesDomain(originalPath, domain) {
|
|
476
|
+
if (!domain.indexable || domain.indexable.length === 0) return true;
|
|
477
|
+
return domain.indexable.some((pat) => matchGlob(originalPath, pat));
|
|
478
|
+
}
|
|
479
|
+
async indexEntry(domain, originalPath, entryPath) {
|
|
480
|
+
let content;
|
|
481
|
+
try {
|
|
482
|
+
const raw = (await this.spaceAfs.read?.(toAbsolute(originalPath)))?.data?.content;
|
|
483
|
+
if (typeof raw !== "string" || raw.length === 0) return "noop";
|
|
484
|
+
try {
|
|
485
|
+
content = JSON.parse(raw);
|
|
486
|
+
} catch {
|
|
487
|
+
content = raw;
|
|
488
|
+
}
|
|
489
|
+
} catch {
|
|
490
|
+
if (await this.entryStillCurrent(originalPath)) throw new ChangelogAbortError(originalPath);
|
|
491
|
+
return "noop";
|
|
492
|
+
}
|
|
493
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
494
|
+
const anchors = [];
|
|
495
|
+
if (domain.fields && typeof content === "object" && content !== null) {
|
|
496
|
+
const fieldValues = applyFieldMap(content, domain.fields);
|
|
497
|
+
for (const [fieldName, values] of Object.entries(fieldValues)) for (const value of values) if (fieldName === "time") anchors.push({
|
|
498
|
+
type: "time",
|
|
499
|
+
confidence: 1,
|
|
500
|
+
timestamp: value
|
|
501
|
+
});
|
|
502
|
+
else anchors.push({
|
|
503
|
+
type: fieldName,
|
|
504
|
+
confidence: 1,
|
|
505
|
+
value
|
|
506
|
+
});
|
|
507
|
+
}
|
|
508
|
+
const entry = {
|
|
509
|
+
entryPath,
|
|
510
|
+
sourcePath: originalPath,
|
|
511
|
+
domain: domain.name,
|
|
512
|
+
state: "active",
|
|
513
|
+
indexedAt: now,
|
|
514
|
+
updatedAt: now,
|
|
515
|
+
anchorCount: anchors.length
|
|
516
|
+
};
|
|
517
|
+
let summary;
|
|
518
|
+
if (domain.summary) {
|
|
519
|
+
summary = extractSummaryText(content, domain.summary);
|
|
520
|
+
if (domain.enrich && typeof content === "object" && content !== null) summary = await applyEnrichContentFile({
|
|
521
|
+
afs: this.spaceAfs,
|
|
522
|
+
itemJson: content,
|
|
523
|
+
baseSummary: summary,
|
|
524
|
+
enrich: domain.enrich,
|
|
525
|
+
scope: domain.scope
|
|
526
|
+
});
|
|
527
|
+
}
|
|
528
|
+
try {
|
|
529
|
+
if (this.backend.writeIndexedDocument) await this.backend.writeIndexedDocument({
|
|
530
|
+
entry,
|
|
531
|
+
anchors,
|
|
532
|
+
summary
|
|
533
|
+
});
|
|
534
|
+
else {
|
|
535
|
+
await this.backend.writeEntry(entry);
|
|
536
|
+
await this.backend.writeAnchors(entryPath, anchors);
|
|
537
|
+
if (summary) await this.backend.writeSummary(entryPath, summary);
|
|
538
|
+
}
|
|
539
|
+
} catch (err) {
|
|
540
|
+
throw new ChangelogAbortError(originalPath, `index write failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
541
|
+
}
|
|
542
|
+
if (summary) {
|
|
543
|
+
if (this.embed && this.backend.writeEmbedding) try {
|
|
544
|
+
const vector = await this.embed(summary);
|
|
545
|
+
if (vector) await this.backend.writeEmbedding(entryPath, vector, domain.name);
|
|
546
|
+
else log.debug("embed skipped (embedder returned null)", { entryPath });
|
|
547
|
+
} catch (err) {
|
|
548
|
+
log.debug("embed failed (entry kept, FTS/anchor only)", {
|
|
549
|
+
entryPath,
|
|
550
|
+
error: err instanceof Error ? err.message : String(err)
|
|
551
|
+
});
|
|
552
|
+
}
|
|
553
|
+
}
|
|
554
|
+
return "indexed";
|
|
555
|
+
}
|
|
556
|
+
/**
|
|
557
|
+
* Build the stable index entry path for a file. Uses the domain scope
|
|
558
|
+
* (default "/user") as a prefix so query-scope filtering works.
|
|
559
|
+
*/
|
|
560
|
+
makeEntryPath(domain, originalPath) {
|
|
561
|
+
return joinURL(domain.scope ?? "/user", originalPath);
|
|
562
|
+
}
|
|
563
|
+
};
|
|
564
|
+
function matchGlob(path, pattern) {
|
|
565
|
+
const regexStr = pattern.replace(/[.+^${}()|[\]\\]/g, "\\$&").replace(/\*\*/g, "⧫").replace(/\*/g, "[^/]*").replace(/⧫/g, ".*").replace(/\?/g, "[^/]");
|
|
566
|
+
return (/* @__PURE__ */ new RegExp(`(^|/)${regexStr}$`)).test(path);
|
|
567
|
+
}
|
|
568
|
+
|
|
569
|
+
//#endregion
|
|
570
|
+
export { DEFAULT_MAX_DIRTY_ROWS_PER_TICK, DEFAULT_MAX_NOOP_PER_TICK, DEFAULT_MAX_PROCESSED_PER_TICK, DEFAULT_MAX_SCAN_PER_TICK, FOLLOWER_CONSUMER_ID, IndexFollower, deriveFollowerConsumerId, userIndexKey };
|
|
571
|
+
//# sourceMappingURL=follower.mjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"follower.mjs","names":[],"sources":["../src/follower.ts"],"sourcesContent":["/**\n * IndexFollower — DRAINS the source-side dirty queue (`ds_dirty_path`, the merged\n * current-state projection of the changelog) and incrementally indexes file\n * writes/deletes into the index backend. Dirty drain is the SOLE mechanism —\n * the changelog `/.changelog` replay was deleted at the §13 cutover; a follower\n * with no `dirtyQueue` is inert.\n *\n * Two entry points:\n * runCatchUp() — inline drain; called from execQuery so a query always sees a\n * fresh index (processes whatever dirty rows are pending).\n * runBatch() — bounded per-tick drain (maxProcessed/maxNoop/maxScan caps);\n * called from the cron task.\n *\n * Cursor: a `consumer_id` row in `idx_index_cursor` registers the follower and\n * carries its instanceDid + domains snapshot for durable cron discovery. Its\n * `last_seq` is a DEAD column post-cutover (nothing advances it during drain —\n * §4.2); scheduling fairness rides `served_at` (markCursorServed). All domains a\n * SINGLE blocklet declares share one cursor; DIFFERENT blocklets under the same\n * caller each get their own (their `/user` data lives in separate instance\n * spaces / dirty-queue prefixes).\n *\n * Consumer-id identity (design.md §4.4 decision-lock addendum, owner\n * 2026-07-05; per-instance fix): a single shared backend hosts MULTIPLE\n * `(scope, instanceDid)` pairs at once — Node's one-shared-SQLite-file rework\n * of #936 P4, and CF's one-shared-D1 model. So the bare `FOLLOWER_CONSUMER_ID`\n * isn't enough to identify ONE follower's cursor row;\n * `deriveFollowerConsumerId(scope, instanceDid)` folds BOTH in\n * (`index:{scope}:{instanceDid}`), the same compositing technique already used\n * for domain-name isolation (`{declared name}:{scope}`). Keying by scope ALONE\n * (the pre-fix behavior) let a caller's SECOND index-blocklet collide on the\n * first's cursor row (`INSERT OR IGNORE` → dropped) so its changelog was never\n * followed. D1 additionally isolates via its own `scope` column (PRIMARY KEY\n * (scope, consumer_id)); both runtimes derive the consumer_id the same way so a\n * persisted row means the same thing everywhere (`IndexBackend.listCursorScopes`\n * matches on the `{FOLLOWER_CONSUMER_ID}:` PREFIX for exactly this reason — see\n * d1-index-backend.ts).\n */\n// makeNsLog comes from @aigne/afs (ambient-based, browser/CF-safe) — NOT\n// @aigne/afs-logger, whose ALS variant pulls node:async_hooks into any bundle\n// that includes this provider (docs/guides/logging.md \"helper 选择\").\nimport { AFSNotFoundError, type AFSRoot, type BlockletIndexDomain, makeNsLog } from \"@aigne/afs\";\nimport { joinURL } from \"ufo\";\nimport type { IndexBackend } from \"./backend/types.js\";\nimport type { DirtyPathQueue } from \"./dirty-queue.js\";\nimport {\n applyEnrichContentFile,\n applyFieldMap,\n extractSummaryText,\n} from \"./extraction/field-map.js\";\nimport type { Anchor, IndexedEntry } from \"./types.js\";\n\nconst log = makeNsLog(\"provider:index\");\n\nexport const FOLLOWER_CONSUMER_ID = \"index\";\n\n/**\n * Derive a follower consumer_id that isolates per `(scope, instanceDid)` —\n * `\"index:{scope}:{instanceDid}\"`. A single shared backend (Node's `/user/index`\n * file, CF's D1) hosts many callers AND many blocklets per caller; the cursor\n * row must be unique per (caller scope × blocklet instance) because each\n * blocklet follows its OWN instance-space changelog. `instanceDid` is optional\n * for backward compatibility (bare `index:{scope}` for pre-fix rows / callers\n * that don't supply it); passing the bare `FOLLOWER_CONSUMER_ID` is only safe\n * for single-scope callers (tests, standalone use).\n */\nexport function deriveFollowerConsumerId(scope: string, instanceDid?: string | null): string {\n return instanceDid\n ? `${FOLLOWER_CONSUMER_ID}:${scope}:${instanceDid}`\n : `${FOLLOWER_CONSUMER_ID}:${scope}`;\n}\n\n/**\n * Memory-map key for a caller's per-blocklet index resources (the AFSIndex\n * handle, its in-flight build promise, its IndexFollower). Isolation is\n * per-`(scope, instanceDid)`: the SAME caller (scope) gets a SEPARATE index per\n * blocklet (instanceDid). Keying these isolate-lived maps by scope ALONE let a\n * caller's second index-blocklet reuse the FIRST's handle/follower (wrong bound\n * domains + wrong instance-space changelog) — the in-memory twin of the cursor\n * collision `deriveFollowerConsumerId` guards against.\n */\nexport function userIndexKey(scope: string, instanceDid?: string | null): string {\n return instanceDid ? `${scope}::${instanceDid}` : scope;\n}\n\n// The ONLY per-tick indexing budget defaults, shared by BOTH entry points:\n// • runCatchUp() — inline on EVERY search (subrequest-sensitive)\n// • runBatch() — cron drain, when the caller passes no explicit limits\n// PROCESSED caps matched PUT attempts (read + FTS writes + ≈1 workers-ai embed +\n// ≈1 Upstash HTTP write ≈ 9 subrequests each) — the real cost axis that must\n// stay under Cloudflare's free-tier ceiling (50 subrequests / invocation). NOOP\n// caps matched cleanup rows known up front not to need content indexing\n// (delete/remove). The third budget bounds how many DIRTY ROWS a tick FETCHES.\n// The CF runtime overrides all three from AFS_INDEX_MAX_*_PER_TICK (cron →\n// runBatch limits; search → setCatchUpLimits). A runtime with no subrequest cap\n// (Node daemon) passes its OWN larger runBatch limits and never relies on these.\nexport const DEFAULT_MAX_PROCESSED_PER_TICK = 4;\nexport const DEFAULT_MAX_NOOP_PER_TICK = 12;\n// The per-tick DIRTY-ROW fetch budget (index-follower-dirty-path). Historically\n// `maxScan` = \"changelog rows walked past\"; after the §13 cutover the changelog\n// scan is gone, so this bounds how many deduped `ds_dirty_path` rows a tick\n// fetches — O(unique dirty paths), not O(changelog rows). The `maxScan` name is\n// KEPT across the budget chain (scheduler/resolver/crons/env) as a stable field\n// key; `maxDirtyRows` is the preferred alias on `runBatch`. (§B naming: the\n// design permits keeping the field + this note over a public-API rename.)\nexport const DEFAULT_MAX_SCAN_PER_TICK = 400;\n/** Preferred alias for {@link DEFAULT_MAX_SCAN_PER_TICK} in dirty-drain call sites. */\nexport const DEFAULT_MAX_DIRTY_ROWS_PER_TICK = DEFAULT_MAX_SCAN_PER_TICK;\n\n// Retry backoff (ms) applied to a dirty row's `next_attempt_at` when its\n// processing fails transiently (ChangelogAbortError). Keeps a poison row out of\n// the ready set for one backoff window so it can't refill the tick's budget —\n// the dirty-mode equivalent of the changelog \"stop-without-advancing\" behavior.\nconst DIRTY_RETRY_BACKOFF_MS = 60_000;\n\n// `AFS#read/stat` require an ABSOLUTE path (validatePath throws otherwise) — a\n// real `spaceAfs` (DIDSpaceProvider-backed) enforces this, unlike a hand-rolled\n// test stub. A dirty row's `path` is the fragment-stripped `originalPath` (no\n// leading slash), so it must be normalized before crossing the AFS boundary.\nfunction toAbsolute(path: string): string {\n return joinURL(\"/\", path);\n}\n\n/**\n * Internal signal thrown by `indexEntry` / the dirty-drain remove path when a\n * read/stat/index failed on content that is STILL current (issue #935 BLOCK\n * fix #3 / sync-engine parity): `processDirty` catches this and BACKS OFF the\n * dirty row (marks it, doesn't ack) so the next tick retries it instead of\n * silently losing the write. (Historically named for the changelog replay that\n * the dirty-path cutover deleted; kept as the dirty-drain retry signal.)\n */\nclass ChangelogAbortError extends Error {\n constructor(originalPath: string, reason = \"transient read failure\") {\n super(`${reason} for \"${originalPath}\" — retry next run`);\n this.name = \"ChangelogAbortError\";\n }\n}\n\nexport interface IndexFollowerBatchResult {\n scanned: number;\n /** Matched PUT rows that consumed the expensive maxProcessed budget. */\n processed: number;\n /**\n * Rows that consumed the maxNoop budget (design §8): delete rows, put-rows\n * whose live state was missing (removed as cleanup), AND non-indexable\n * (unmatched-domain) rows.\n */\n noopProcessed: number;\n /** PUT rows that actually wrote entry/anchor/summary index data. */\n indexed: number;\n /** Delete/remove rows whose index cleanup was attempted. */\n deleted: number;\n /** Matched rows that intentionally wrote no index data (empty or stale content). */\n nooped: number;\n advanced: boolean;\n hasMore: boolean;\n /**\n * At least one dirty row lost the ack race-guard this tick — a newer write\n * bumped `latest_seq` while we processed, so the guarded ack no-oped and the\n * newer dirty row waits for the next tick. NOT an error (real progress was\n * made on the superseded state); surfaced so the CF/Node drain logs can\n * distinguish healthy churn (`ackRace`) from failures (`failed`).\n */\n ackRace: boolean;\n /**\n * Why the tick stopped. `scan_budget` = the dirty-row FETCH budget (maxScan /\n * maxDirtyRows) was exhausted — the name predates the §13 cutover (it used to\n * mean \"changelog scan budget\"); kept as a stable value across the exported\n * scheduler/daemon types.\n */\n stopReason?: \"processed_budget\" | \"noop_budget\" | \"scan_budget\" | \"exhausted\" | \"aborted\";\n /** Log-compat only; dirty mode never advances a seq cursor (both 0). */\n fromSeq: number;\n toSeq: number;\n matched: number;\n unmatched: number;\n skipped: number;\n failed: number;\n aborted: boolean;\n abortReason?: string;\n}\n\nexport interface IndexFollowerOptions {\n /** AFS backed by DIDSpaceProvider+PrefixedTreeIndex — CURRENT-state file read/stat\n * for the dirty drain (the `.changelog` consumption line was deleted at §13 cutover). */\n spaceAfs: AFSRoot;\n /** Index backend for writing indexed data. Must implement cursor methods. */\n backend: IndexBackend;\n /** Domain declarations from blocklet.yaml `index:` section. */\n domains: BlockletIndexDomain[];\n /**\n * Cursor row identity for THIS follower. Defaults to the bare\n * `FOLLOWER_CONSUMER_ID` (\"index\") for single-scope callers (tests,\n * standalone use). Any caller whose backend hosts MULTIPLE scopes MUST pass\n * `deriveFollowerConsumerId(scope)` — otherwise every scope sharing that\n * backend would read/write the SAME `idx_index_cursor` row, each run\n * clobbering the others' progress.\n */\n consumerId?: string;\n /**\n * Optional passage embedder — generates the vector stored alongside each\n * indexed summary for semantic recall (Epic #932). Usually left unset here and\n * wired later via `setEmbedder` (see its doc). Returns null to skip embedding.\n */\n embed?: (text: string) => Promise<number[] | null>;\n /**\n * Source-side dirty queue (index-follower-dirty-path). The follower DRAINS the\n * merged `ds_dirty_path` projection — catch-up is O(unique dirty paths), not\n * O(changelog rows). This is the SOLE catch-up mechanism (§13 cutover — the\n * changelog `/.changelog` replay was deleted); a follower with no dirty queue\n * has no source to drain and `runCatchUp`/`runBatch` are no-ops for it.\n * Normally a fragment-scoped `PrefixedTreeIndex` so `listDirtyPaths` returns\n * stripped `originalPath`s. ack/fail must be a runtime-internal capability,\n * never a public `/user` action.\n */\n dirtyQueue?: DirtyPathQueue;\n}\n\nexport class IndexFollower {\n private readonly spaceAfs: AFSRoot;\n private readonly backend: IndexBackend;\n private readonly domains: BlockletIndexDomain[];\n private readonly consumerId: string;\n private embed?: (text: string) => Promise<number[] | null>;\n // `maxScan` here is the per-tick dirty-row FETCH budget (kept as the field key\n // across the whole budget chain; §B naming). See DEFAULT_MAX_SCAN_PER_TICK.\n private catchUpLimits?: { maxProcessed: number; maxNoop?: number; maxScan: number };\n private readonly dirtyQueue?: DirtyPathQueue;\n\n constructor(opts: IndexFollowerOptions) {\n this.spaceAfs = opts.spaceAfs;\n this.backend = opts.backend;\n this.domains = opts.domains;\n this.consumerId = opts.consumerId ?? FOLLOWER_CONSUMER_ID;\n this.embed = opts.embed;\n this.dirtyQueue = opts.dirtyQueue;\n }\n\n /**\n * Override the inline `runCatchUp()` budget — the per-invocation caps on\n * matched PUT attempts, matched delete/remove rows, and dirty rows FETCHED\n * (`maxScan`). Injected by the runtime from the SAME env group the cron uses\n * (`AFS_INDEX_MAX_*_PER_TICK`) so both indexing paths share one budget group.\n * Unset → provider defaults (standalone/tests).\n */\n setCatchUpLimits(limits: { maxProcessed: number; maxNoop?: number; maxScan: number }): void {\n this.catchUpLimits = limits;\n }\n\n /**\n * Wire (or replace) the passage embedder used at index time — semantic recall\n * (Epic #932). Called by `AFSIndex.setFollower` (query-path follower) and by\n * `rebuildFollowerFromCursor` (CF cron-path follower) so BOTH indexing paths\n * write embeddings; without this a changelog-indexed entry has only FTS +\n * anchors and cross-lingual / semantic queries can't match it. A no-op\n * embedder (or a backend without `writeEmbedding`) degrades cleanly to\n * FTS+anchor.\n */\n setEmbedder(embed: (text: string) => Promise<number[] | null>): void {\n this.embed = embed;\n }\n\n /**\n * Lightweight catch-up: drain any ready dirty rows so a query sees a fresh\n * index without waiting for the next cron tick. Called from execQuery.\n *\n * Dirty drain is the SOLE catch-up mechanism (index-follower-dirty-path §13\n * cutover — the changelog `/.changelog` replay was deleted). Freshness = \"are\n * there ready dirty rows under my prefix\", so there is no `last_seq` probe: a\n * query sees a fresh index by processing whatever dirty paths are pending\n * (design §4.2, §8). A follower with no dirty queue (standalone/legacy) is a\n * no-op here — it has no source to drain.\n */\n async runCatchUp(): Promise<void> {\n if (!this.dirtyQueue) return;\n await this.processDirty({\n maxProcessed: this.catchUpLimits?.maxProcessed ?? DEFAULT_MAX_PROCESSED_PER_TICK,\n maxNoop: this.catchUpLimits?.maxNoop ?? DEFAULT_MAX_NOOP_PER_TICK,\n maxDirtyRows: this.catchUpLimits?.maxScan ?? DEFAULT_MAX_DIRTY_ROWS_PER_TICK,\n });\n }\n\n /**\n * Bounded per-tick drain of the dirty queue — attempt up to `maxProcessed`\n * matched PUT rows, up to `maxNoop` matched delete/remove/noop rows, fetching\n * at most `maxScan` (a.k.a. `maxDirtyRows`) dirty rows. Called from the cron.\n *\n * Dirty drain is the SOLE mechanism (index-follower-dirty-path §13 cutover —\n * changelog replay was deleted). A follower with no dirty queue has no source\n * to drain, so it is idle. Separate budgets are intentional: matched PUT rows\n * are the expensive read+embed+write axis, delete/noop rows are cheaper.\n */\n async runBatch(limits?: {\n maxProcessed?: number;\n maxNoop?: number;\n /** Max dirty rows fetched this tick. `maxDirtyRows` is the preferred name. */\n maxScan?: number;\n maxDirtyRows?: number;\n }): Promise<IndexFollowerBatchResult> {\n if (!this.dirtyQueue) return this.idleResult();\n return this.processDirty({\n maxProcessed: limits?.maxProcessed ?? DEFAULT_MAX_PROCESSED_PER_TICK,\n maxNoop: limits?.maxNoop ?? DEFAULT_MAX_NOOP_PER_TICK,\n maxDirtyRows: limits?.maxDirtyRows ?? limits?.maxScan ?? DEFAULT_MAX_DIRTY_ROWS_PER_TICK,\n });\n }\n\n /** Empty batch result (no dirty queue wired / nothing to drain). */\n private idleResult(): IndexFollowerBatchResult {\n return {\n scanned: 0,\n processed: 0,\n noopProcessed: 0,\n indexed: 0,\n deleted: 0,\n nooped: 0,\n advanced: false,\n hasMore: false,\n ackRace: false,\n stopReason: \"exhausted\",\n fromSeq: 0,\n toSeq: 0,\n matched: 0,\n unmatched: 0,\n skipped: 0,\n failed: 0,\n aborted: false,\n };\n }\n\n /**\n * Cold rebuild of an EXPLICIT list of space-relative paths — the enumeration\n * is done by the caller (an external driver listing the live tree via D1),\n * NOT in this process. Reads CURRENT live state per path and writes the\n * derived index layers, independent of any queue — so it is the seam the\n * offline reset-and-rebuild (`offlineRebuildScope`) uses to rebuild from live\n * `ds_entries`. Keeping the per-call cost to just `paths.length`\n * read+embed+writes with ZERO in-Worker tree enumeration is what keeps a small\n * batch inside the CF free-tier per-request CPU/subrequest budget (an in-Worker\n * `maxDepth:-1` enumeration pushed cold isolates over that budget into 503s).\n * Each path is matched against the declared domains and indexed through the\n * same `indexEntry` as the drain path (byte-identical output). `paths` are\n * space-relative, e.g. `\"items/foo.json\"` (leading slash tolerated).\n */\n async reindexPaths(\n paths: string[],\n onProgress?: (done: number, total: number) => void,\n ): Promise<{ indexed: number; failed: number; skipped: number }> {\n let indexed = 0;\n let failed = 0;\n let skipped = 0;\n let done = 0;\n for (const raw of paths) {\n const rel = raw.replace(/^\\/+/, \"\");\n let matched = false;\n for (const domain of this.domains) {\n if (!this.pathMatchesDomain(rel, domain)) continue;\n matched = true;\n try {\n const result = await this.indexEntry(domain, rel, this.makeEntryPath(domain, rel));\n if (result === \"indexed\") indexed++;\n } catch (err) {\n failed++;\n log.warn(`reindex failed: ${rel}`, err instanceof Error ? err.message : err);\n }\n }\n if (!matched) skipped++;\n done++;\n if (onProgress && (done % 50 === 0 || done === paths.length)) onProgress(done, paths.length);\n }\n return { indexed, failed, skipped };\n }\n\n /**\n * Dirty drain (index-follower-dirty-path §8) — the O(unique dirty paths)\n * replacement for changelog replay. Fetches ready dirty rows under this\n * follower's prefix, and for each: domain-matches the stripped `originalPath`,\n * reads CURRENT live state, and indexes / removes accordingly, then acks with\n * the `latestSeq` race guard. A transient failure marks the row with a backoff\n * (so a poison row can't refill the tick) instead of acking.\n *\n * Dirty mode NEVER writes `idx_index_cursor.last_seq` (design §4.2 — it is a\n * dead column post-cutover); scheduling reads `advanced/hasMore/scanned` +\n * the dirty count. The `dirtyQueue.listDirtyPaths({ prefix: \"\" })` call assumes\n * a fragment-scoped queue (PrefixedTreeIndex), so `row.path` is already the\n * stripped `originalPath` the domain globs + `spaceAfs.read` expect.\n */\n private async processDirty(limits: {\n maxProcessed: number;\n maxNoop: number;\n maxDirtyRows: number;\n }): Promise<IndexFollowerBatchResult> {\n const result0 = (): IndexFollowerBatchResult => ({\n scanned: 0,\n processed: 0,\n noopProcessed: 0,\n indexed: 0,\n deleted: 0,\n nooped: 0,\n advanced: false,\n hasMore: false,\n ackRace: false,\n stopReason: \"exhausted\",\n fromSeq: 0,\n toSeq: 0,\n matched: 0,\n unmatched: 0,\n skipped: 0,\n failed: 0,\n aborted: false,\n });\n const dirtyQueue = this.dirtyQueue;\n if (!dirtyQueue) return result0();\n const maxDirtyRows = limits.maxDirtyRows > 0 ? limits.maxDirtyRows : 0;\n if (maxDirtyRows <= 0 || (limits.maxProcessed <= 0 && limits.maxNoop <= 0)) return result0();\n\n let processedRemaining = limits.maxProcessed;\n let noopRemaining = limits.maxNoop;\n let scanned = 0;\n let processed = 0;\n let noopProcessed = 0;\n let indexed = 0;\n let deleted = 0;\n let nooped = 0;\n let matched = 0;\n let unmatched = 0;\n let failed = 0;\n let advanced = false;\n let hasMore = false;\n let ackRace = false;\n let stopReason: IndexFollowerBatchResult[\"stopReason\"] | undefined;\n\n const now = Date.now();\n const rows = await dirtyQueue.listDirtyPaths({ prefix: \"\", limit: maxDirtyRows, now });\n\n for (const row of rows) {\n scanned++;\n const matchedDomains = this.domains.filter((d) => this.pathMatchesDomain(row.path, d));\n\n // No declared domain claims this path — ack it as a noop. Metered on the\n // noop budget and counted in noopProcessed (design §8: non-indexable paths\n // count as noop so they can't linger in the queue nor be processed\n // unbounded per tick).\n if (matchedDomains.length === 0) {\n if (noopRemaining <= 0) {\n hasMore = true;\n stopReason = \"noop_budget\";\n break;\n }\n unmatched++;\n noopRemaining--;\n noopProcessed++;\n const ok = await dirtyQueue.ackDirtyPath(row.path, row.latestSeq);\n if (ok) advanced = true;\n else {\n hasMore = true;\n ackRace = true;\n }\n continue;\n }\n\n const isDelete = row.op === \"delete\";\n // Budget gate by op class — a matched PUT costs the expensive\n // read+index+embed budget, a delete the cheaper noop budget.\n if (isDelete) {\n if (noopRemaining <= 0) {\n hasMore = true;\n stopReason = \"noop_budget\";\n break;\n }\n } else if (processedRemaining <= 0) {\n hasMore = true;\n stopReason = \"processed_budget\";\n break;\n }\n\n try {\n let didIndex = 0;\n let didRemove = 0;\n if (isDelete) {\n await this.removeDirty(matchedDomains, row.path);\n didRemove = 1;\n } else {\n const r = await this.indexPutDirty(matchedDomains, row.path);\n didIndex = r.indexed;\n didRemove = r.removed;\n }\n matched++;\n if (isDelete) {\n noopRemaining--;\n noopProcessed++;\n deleted++;\n } else if (didIndex > 0) {\n processedRemaining--;\n processed++;\n indexed++;\n } else if (didRemove > 0) {\n // stale put + live missing → removed as cleanup (design §8 step 6)\n noopProcessed++;\n deleted++;\n } else {\n nooped++;\n }\n const ok = await dirtyQueue.ackDirtyPath(row.path, row.latestSeq);\n if (ok) {\n advanced = true;\n } else {\n // Guard lost — a newer write bumped latest_seq while we processed.\n // We DID make real progress on the (now superseded) state, so this\n // counts as advanced (design §8); the newer dirty row waits for the\n // next tick.\n advanced = true;\n hasMore = true;\n ackRace = true;\n }\n } catch (err) {\n // Transient failure (read/stat/index blip, or a replica-consistency\n // miss surfaced as a non-404 error) → back the row off instead of\n // acking, so the next tick retries the CURRENT state (design §8 step 0).\n failed++;\n const msg = err instanceof Error ? err.message : String(err);\n await dirtyQueue.markDirtyPathError(\n row.path,\n row.latestSeq,\n msg,\n now + DIRTY_RETRY_BACKOFF_MS,\n );\n log.warn(`dirty drain failed (backed off): ${row.path}`, msg);\n }\n\n if (processedRemaining <= 0 && noopRemaining <= 0) {\n hasMore = true;\n stopReason = \"processed_budget\";\n break;\n }\n }\n\n if (!stopReason) {\n // Finished every fetched row within budget. If we filled the fetch page a\n // newer batch may remain; probe one ready row to set hasMore honestly.\n if (rows.length >= maxDirtyRows && maxDirtyRows > 0) {\n const more = await dirtyQueue.listDirtyPaths({ prefix: \"\", limit: 1, now });\n if (more.length > 0) hasMore = true;\n }\n stopReason = hasMore ? \"scan_budget\" : \"exhausted\";\n }\n\n log.debug(\"dirty drain processed\", {\n consumerId: this.consumerId,\n scanned,\n processed,\n noopProcessed,\n indexed,\n deleted,\n nooped,\n matched,\n unmatched,\n failed,\n advanced,\n hasMore,\n ackRace,\n stopReason,\n });\n\n return {\n scanned,\n processed,\n noopProcessed,\n indexed,\n deleted,\n nooped,\n advanced,\n hasMore,\n ackRace,\n stopReason,\n // Dirty mode never advances a seq cursor — 0 is a log-compat placeholder.\n fromSeq: 0,\n toSeq: 0,\n matched,\n unmatched,\n skipped: 0,\n failed,\n aborted: false,\n };\n }\n\n /** Remove the indexed doc(s) for a delete/removed dirty row across matched domains. */\n private async removeDirty(\n matchedDomains: BlockletIndexDomain[],\n originalPath: string,\n ): Promise<void> {\n for (const domain of matchedDomains) {\n await this.removeIndexed(this.makeEntryPath(domain, originalPath), originalPath);\n }\n }\n\n /** Shared index-removal (delete branch) — throws ChangelogAbortError on failure so the row backs off. */\n private async removeIndexed(entryPath: string, originalPath: string): Promise<void> {\n try {\n if (this.backend.removeIndexedDocument) {\n await this.backend.removeIndexedDocument(entryPath);\n } else {\n await this.backend.removeAnchors(entryPath);\n await this.backend.removeSummary(entryPath);\n await this.backend.removeEntry(entryPath);\n }\n } catch (err) {\n throw new ChangelogAbortError(\n originalPath,\n `index delete failed: ${err instanceof Error ? err.message : String(err)}`,\n );\n }\n }\n\n /**\n * Handle a `put` dirty row against CURRENT live state (design §4.3 / §8 step 6):\n * - live exists → indexEntry (writes FTS/anchor/summary/embedding).\n * - live missing (authoritative 404) → remove the indexed doc.\n * - stat throws a NON-404 error → transient → ChangelogAbortError (retry).\n * This is what makes a stale put whose file was deleted converge to \"removed\",\n * while a replica-lag / D1 blip is NEVER mistaken for a deletion.\n */\n private async indexPutDirty(\n matchedDomains: BlockletIndexDomain[],\n originalPath: string,\n ): Promise<{ indexed: number; removed: number }> {\n let live: unknown;\n try {\n const statResult = await this.spaceAfs.stat?.(toAbsolute(originalPath));\n live = statResult?.data;\n } catch (err) {\n if (err instanceof AFSNotFoundError) {\n live = undefined; // authoritative miss — fall through to remove below\n } else {\n throw new ChangelogAbortError(\n originalPath,\n `stat failed: ${err instanceof Error ? err.message : String(err)}`,\n );\n }\n }\n if (!live) {\n await this.removeDirty(matchedDomains, originalPath);\n return { indexed: 0, removed: 1 };\n }\n let indexed = 0;\n for (const domain of matchedDomains) {\n // We already confirmed live existence above; dirty mode's correctness rests\n // on the live-read (newest content) + the ack race-guard (a later write\n // bumps latest_seq → the guarded ack no-ops → the newer dirty row is\n // reprocessed), NOT on a cid-supersession check. So indexEntry does ONE\n // live read per merged put — no redundant supersession stat.\n const r = await this.indexEntry(\n domain,\n originalPath,\n this.makeEntryPath(domain, originalPath),\n );\n if (r === \"indexed\") indexed++;\n }\n return { indexed, removed: 0 };\n }\n\n /**\n * Does `originalPath` STILL EXIST (via `stat`)? Used to classify a `read()`\n * failure (issue #935 BLOCK fix #3 / sync-engine parity):\n * - stat 404 (or throws) → the file is gone (later delete, or never existed\n * at this position) → the row is stale, safe to skip.\n * - stat succeeds → the file is live; the read failure was transient (R2/D1\n * blip) and must be retried, not lost.\n * Post-cutover the drain only ever indexes live state (cid is never compared —\n * supersession is caught by the ack race-guard), so existence is the only\n * question left here.\n */\n private async entryStillCurrent(originalPath: string): Promise<boolean> {\n try {\n const result = await this.spaceAfs.stat?.(toAbsolute(originalPath));\n return !!result?.data;\n } catch {\n return false;\n }\n }\n\n private pathMatchesDomain(originalPath: string, domain: BlockletIndexDomain): boolean {\n if (!domain.indexable || domain.indexable.length === 0) return true;\n return domain.indexable.some((pat) => matchGlob(originalPath, pat));\n }\n\n private async indexEntry(\n domain: BlockletIndexDomain,\n originalPath: string,\n entryPath: string,\n ): Promise<\"indexed\" | \"noop\"> {\n let content: unknown;\n try {\n const result = await this.spaceAfs.read?.(toAbsolute(originalPath));\n const raw = result?.data?.content;\n if (typeof raw !== \"string\" || raw.length === 0) return \"noop\";\n try {\n content = JSON.parse(raw);\n } catch {\n content = raw;\n }\n } catch {\n // Classify the read failure before deciding retry vs skip — see\n // `entryStillCurrent` doc (issue #935 BLOCK fix #3).\n if (await this.entryStillCurrent(originalPath)) {\n throw new ChangelogAbortError(originalPath);\n }\n // Gone — this row is stale, safe to skip.\n return \"noop\";\n }\n\n const now = new Date().toISOString();\n const anchors: Anchor[] = [];\n\n if (domain.fields && typeof content === \"object\" && content !== null) {\n const fieldValues = applyFieldMap(\n content,\n domain.fields as Record<string, string | string[]>,\n );\n for (const [fieldName, values] of Object.entries(fieldValues)) {\n for (const value of values) {\n if (fieldName === \"time\") {\n anchors.push({ type: \"time\", confidence: 1.0, timestamp: value });\n } else {\n anchors.push({ type: fieldName, confidence: 1.0, value });\n }\n }\n }\n }\n\n const entry: IndexedEntry = {\n entryPath,\n sourcePath: originalPath,\n domain: domain.name,\n state: \"active\",\n indexedAt: now,\n updatedAt: now,\n anchorCount: anchors.length,\n };\n let summary: string | undefined;\n if (domain.summary) {\n summary = extractSummaryText(content, domain.summary as string | string[]);\n if (domain.enrich && typeof content === \"object\" && content !== null) {\n summary = await applyEnrichContentFile({\n afs: this.spaceAfs,\n itemJson: content as Record<string, unknown>,\n baseSummary: summary,\n enrich: domain.enrich,\n scope: domain.scope,\n });\n }\n }\n\n try {\n if (this.backend.writeIndexedDocument) {\n await this.backend.writeIndexedDocument({\n entry,\n anchors,\n summary,\n });\n } else {\n await this.backend.writeEntry(entry);\n await this.backend.writeAnchors(entryPath, anchors);\n if (summary) await this.backend.writeSummary(entryPath, summary);\n }\n } catch (err) {\n throw new ChangelogAbortError(\n originalPath,\n `index write failed: ${err instanceof Error ? err.message : String(err)}`,\n );\n }\n\n if (summary) {\n // Semantic recall (Epic #932): store the summary's embedding vector so\n // cross-lingual / semantic queries can match this entry. Mirrors what\n // the explicit `index` action (AFSIndex.execIndex) already does — this\n // brings the changelog-follower path to parity so items indexed the\n // normal (event-driven) way ALSO get embeddings, not just FTS+anchors.\n // No-op when no embedder is wired or the backend has no vector column.\n if (this.embed && this.backend.writeEmbedding) {\n try {\n const vector = await this.embed(summary);\n // Pass the entry's domain so vector backends that store metadata\n // (Upstash/Vectorize) can enforce the query-time domain filter —\n // without it, boundDomains queries match nothing.\n if (vector) await this.backend.writeEmbedding(entryPath, vector, domain.name);\n else log.debug(\"embed skipped (embedder returned null)\", { entryPath });\n } catch (err) {\n // Embedding is a best-effort enrichment — a failed embed must never\n // drop the FTS/anchor index write that already succeeded above. But\n // a silent miss here is the #1 cause of \"semantic search can't find\n // this item\", so leave a trace.\n log.debug(\"embed failed (entry kept, FTS/anchor only)\", {\n entryPath,\n error: err instanceof Error ? err.message : String(err),\n });\n }\n }\n }\n return \"indexed\";\n }\n\n /**\n * Build the stable index entry path for a file. Uses the domain scope\n * (default \"/user\") as a prefix so query-scope filtering works.\n */\n private makeEntryPath(domain: BlockletIndexDomain, originalPath: string): string {\n const base = domain.scope ?? \"/user\";\n return joinURL(base, originalPath);\n }\n}\n\nfunction matchGlob(path: string, pattern: string): boolean {\n const regexStr = pattern\n .replace(/[.+^${}()|[\\]\\\\]/g, \"\\\\$&\")\n .replace(/\\*\\*/g, \"⧫\")\n .replace(/\\*/g, \"[^/]*\")\n .replace(/⧫/g, \".*\")\n .replace(/\\?/g, \"[^/]\");\n return new RegExp(`(^|/)${regexStr}$`).test(path);\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAmDA,MAAM,MAAM,UAAU,iBAAiB;AAEvC,MAAa,uBAAuB;;;;;;;;;;;AAYpC,SAAgB,yBAAyB,OAAe,aAAqC;AAC3F,QAAO,cACH,GAAG,qBAAqB,GAAG,MAAM,GAAG,gBACpC,GAAG,qBAAqB,GAAG;;;;;;;;;;;AAYjC,SAAgB,aAAa,OAAe,aAAqC;AAC/E,QAAO,cAAc,GAAG,MAAM,IAAI,gBAAgB;;AAcpD,MAAa,iCAAiC;AAC9C,MAAa,4BAA4B;AAQzC,MAAa,4BAA4B;;AAEzC,MAAa,kCAAkC;AAM/C,MAAM,yBAAyB;AAM/B,SAAS,WAAW,MAAsB;AACxC,QAAO,QAAQ,KAAK,KAAK;;;;;;;;;;AAW3B,IAAM,sBAAN,cAAkC,MAAM;CACtC,YAAY,cAAsB,SAAS,0BAA0B;AACnE,QAAM,GAAG,OAAO,QAAQ,aAAa,oBAAoB;AACzD,OAAK,OAAO;;;AAoFhB,IAAa,gBAAb,MAA2B;CACzB,AAAiB;CACjB,AAAiB;CACjB,AAAiB;CACjB,AAAiB;CACjB,AAAQ;CAGR,AAAQ;CACR,AAAiB;CAEjB,YAAY,MAA4B;AACtC,OAAK,WAAW,KAAK;AACrB,OAAK,UAAU,KAAK;AACpB,OAAK,UAAU,KAAK;AACpB,OAAK,aAAa,KAAK,cAAc;AACrC,OAAK,QAAQ,KAAK;AAClB,OAAK,aAAa,KAAK;;;;;;;;;CAUzB,iBAAiB,QAA2E;AAC1F,OAAK,gBAAgB;;;;;;;;;;;CAYvB,YAAY,OAAyD;AACnE,OAAK,QAAQ;;;;;;;;;;;;;CAcf,MAAM,aAA4B;AAChC,MAAI,CAAC,KAAK,WAAY;AACtB,QAAM,KAAK,aAAa;GACtB,cAAc,KAAK,eAAe,gBAAgB;GAClD,SAAS,KAAK,eAAe,WAAW;GACxC,cAAc,KAAK,eAAe,WAAW;GAC9C,CAAC;;;;;;;;;;;;CAaJ,MAAM,SAAS,QAMuB;AACpC,MAAI,CAAC,KAAK,WAAY,QAAO,KAAK,YAAY;AAC9C,SAAO,KAAK,aAAa;GACvB,cAAc,QAAQ,gBAAgB;GACtC,SAAS,QAAQ,WAAW;GAC5B,cAAc,QAAQ,gBAAgB,QAAQ,WAAW;GAC1D,CAAC;;;CAIJ,AAAQ,aAAuC;AAC7C,SAAO;GACL,SAAS;GACT,WAAW;GACX,eAAe;GACf,SAAS;GACT,SAAS;GACT,QAAQ;GACR,UAAU;GACV,SAAS;GACT,SAAS;GACT,YAAY;GACZ,SAAS;GACT,OAAO;GACP,SAAS;GACT,WAAW;GACX,SAAS;GACT,QAAQ;GACR,SAAS;GACV;;;;;;;;;;;;;;;;CAiBH,MAAM,aACJ,OACA,YAC+D;EAC/D,IAAI,UAAU;EACd,IAAI,SAAS;EACb,IAAI,UAAU;EACd,IAAI,OAAO;AACX,OAAK,MAAM,OAAO,OAAO;GACvB,MAAM,MAAM,IAAI,QAAQ,QAAQ,GAAG;GACnC,IAAI,UAAU;AACd,QAAK,MAAM,UAAU,KAAK,SAAS;AACjC,QAAI,CAAC,KAAK,kBAAkB,KAAK,OAAO,CAAE;AAC1C,cAAU;AACV,QAAI;AAEF,SADe,MAAM,KAAK,WAAW,QAAQ,KAAK,KAAK,cAAc,QAAQ,IAAI,CAAC,KACnE,UAAW;aACnB,KAAK;AACZ;AACA,SAAI,KAAK,mBAAmB,OAAO,eAAe,QAAQ,IAAI,UAAU,IAAI;;;AAGhF,OAAI,CAAC,QAAS;AACd;AACA,OAAI,eAAe,OAAO,OAAO,KAAK,SAAS,MAAM,QAAS,YAAW,MAAM,MAAM,OAAO;;AAE9F,SAAO;GAAE;GAAS;GAAQ;GAAS;;;;;;;;;;;;;;;;CAiBrC,MAAc,aAAa,QAIW;EACpC,MAAM,iBAA2C;GAC/C,SAAS;GACT,WAAW;GACX,eAAe;GACf,SAAS;GACT,SAAS;GACT,QAAQ;GACR,UAAU;GACV,SAAS;GACT,SAAS;GACT,YAAY;GACZ,SAAS;GACT,OAAO;GACP,SAAS;GACT,WAAW;GACX,SAAS;GACT,QAAQ;GACR,SAAS;GACV;EACD,MAAM,aAAa,KAAK;AACxB,MAAI,CAAC,WAAY,QAAO,SAAS;EACjC,MAAM,eAAe,OAAO,eAAe,IAAI,OAAO,eAAe;AACrE,MAAI,gBAAgB,KAAM,OAAO,gBAAgB,KAAK,OAAO,WAAW,EAAI,QAAO,SAAS;EAE5F,IAAI,qBAAqB,OAAO;EAChC,IAAI,gBAAgB,OAAO;EAC3B,IAAI,UAAU;EACd,IAAI,YAAY;EAChB,IAAI,gBAAgB;EACpB,IAAI,UAAU;EACd,IAAI,UAAU;EACd,IAAI,SAAS;EACb,IAAI,UAAU;EACd,IAAI,YAAY;EAChB,IAAI,SAAS;EACb,IAAI,WAAW;EACf,IAAI,UAAU;EACd,IAAI,UAAU;EACd,IAAI;EAEJ,MAAM,MAAM,KAAK,KAAK;EACtB,MAAM,OAAO,MAAM,WAAW,eAAe;GAAE,QAAQ;GAAI,OAAO;GAAc;GAAK,CAAC;AAEtF,OAAK,MAAM,OAAO,MAAM;AACtB;GACA,MAAM,iBAAiB,KAAK,QAAQ,QAAQ,MAAM,KAAK,kBAAkB,IAAI,MAAM,EAAE,CAAC;AAMtF,OAAI,eAAe,WAAW,GAAG;AAC/B,QAAI,iBAAiB,GAAG;AACtB,eAAU;AACV,kBAAa;AACb;;AAEF;AACA;AACA;AAEA,QADW,MAAM,WAAW,aAAa,IAAI,MAAM,IAAI,UAAU,CACzD,YAAW;SACd;AACH,eAAU;AACV,eAAU;;AAEZ;;GAGF,MAAM,WAAW,IAAI,OAAO;AAG5B,OAAI,UACF;QAAI,iBAAiB,GAAG;AACtB,eAAU;AACV,kBAAa;AACb;;cAEO,sBAAsB,GAAG;AAClC,cAAU;AACV,iBAAa;AACb;;AAGF,OAAI;IACF,IAAI,WAAW;IACf,IAAI,YAAY;AAChB,QAAI,UAAU;AACZ,WAAM,KAAK,YAAY,gBAAgB,IAAI,KAAK;AAChD,iBAAY;WACP;KACL,MAAM,IAAI,MAAM,KAAK,cAAc,gBAAgB,IAAI,KAAK;AAC5D,gBAAW,EAAE;AACb,iBAAY,EAAE;;AAEhB;AACA,QAAI,UAAU;AACZ;AACA;AACA;eACS,WAAW,GAAG;AACvB;AACA;AACA;eACS,YAAY,GAAG;AAExB;AACA;UAEA;AAGF,QADW,MAAM,WAAW,aAAa,IAAI,MAAM,IAAI,UAAU,CAE/D,YAAW;SACN;AAKL,gBAAW;AACX,eAAU;AACV,eAAU;;YAEL,KAAK;AAIZ;IACA,MAAM,MAAM,eAAe,QAAQ,IAAI,UAAU,OAAO,IAAI;AAC5D,UAAM,WAAW,mBACf,IAAI,MACJ,IAAI,WACJ,KACA,MAAM,uBACP;AACD,QAAI,KAAK,oCAAoC,IAAI,QAAQ,IAAI;;AAG/D,OAAI,sBAAsB,KAAK,iBAAiB,GAAG;AACjD,cAAU;AACV,iBAAa;AACb;;;AAIJ,MAAI,CAAC,YAAY;AAGf,OAAI,KAAK,UAAU,gBAAgB,eAAe,GAEhD;SADa,MAAM,WAAW,eAAe;KAAE,QAAQ;KAAI,OAAO;KAAG;KAAK,CAAC,EAClE,SAAS,EAAG,WAAU;;AAEjC,gBAAa,UAAU,gBAAgB;;AAGzC,MAAI,MAAM,yBAAyB;GACjC,YAAY,KAAK;GACjB;GACA;GACA;GACA;GACA;GACA;GACA;GACA;GACA;GACA;GACA;GACA;GACA;GACD,CAAC;AAEF,SAAO;GACL;GACA;GACA;GACA;GACA;GACA;GACA;GACA;GACA;GACA;GAEA,SAAS;GACT,OAAO;GACP;GACA;GACA,SAAS;GACT;GACA,SAAS;GACV;;;CAIH,MAAc,YACZ,gBACA,cACe;AACf,OAAK,MAAM,UAAU,eACnB,OAAM,KAAK,cAAc,KAAK,cAAc,QAAQ,aAAa,EAAE,aAAa;;;CAKpF,MAAc,cAAc,WAAmB,cAAqC;AAClF,MAAI;AACF,OAAI,KAAK,QAAQ,sBACf,OAAM,KAAK,QAAQ,sBAAsB,UAAU;QAC9C;AACL,UAAM,KAAK,QAAQ,cAAc,UAAU;AAC3C,UAAM,KAAK,QAAQ,cAAc,UAAU;AAC3C,UAAM,KAAK,QAAQ,YAAY,UAAU;;WAEpC,KAAK;AACZ,SAAM,IAAI,oBACR,cACA,wBAAwB,eAAe,QAAQ,IAAI,UAAU,OAAO,IAAI,GACzE;;;;;;;;;;;CAYL,MAAc,cACZ,gBACA,cAC+C;EAC/C,IAAI;AACJ,MAAI;AAEF,WADmB,MAAM,KAAK,SAAS,OAAO,WAAW,aAAa,CAAC,GACpD;WACZ,KAAK;AACZ,OAAI,eAAe,iBACjB,QAAO;OAEP,OAAM,IAAI,oBACR,cACA,gBAAgB,eAAe,QAAQ,IAAI,UAAU,OAAO,IAAI,GACjE;;AAGL,MAAI,CAAC,MAAM;AACT,SAAM,KAAK,YAAY,gBAAgB,aAAa;AACpD,UAAO;IAAE,SAAS;IAAG,SAAS;IAAG;;EAEnC,IAAI,UAAU;AACd,OAAK,MAAM,UAAU,eAWnB,KALU,MAAM,KAAK,WACnB,QACA,cACA,KAAK,cAAc,QAAQ,aAAa,CACzC,KACS,UAAW;AAEvB,SAAO;GAAE;GAAS,SAAS;GAAG;;;;;;;;;;;;;CAchC,MAAc,kBAAkB,cAAwC;AACtE,MAAI;AAEF,UAAO,CAAC,EADO,MAAM,KAAK,SAAS,OAAO,WAAW,aAAa,CAAC,GAClD;UACX;AACN,UAAO;;;CAIX,AAAQ,kBAAkB,cAAsB,QAAsC;AACpF,MAAI,CAAC,OAAO,aAAa,OAAO,UAAU,WAAW,EAAG,QAAO;AAC/D,SAAO,OAAO,UAAU,MAAM,QAAQ,UAAU,cAAc,IAAI,CAAC;;CAGrE,MAAc,WACZ,QACA,cACA,WAC6B;EAC7B,IAAI;AACJ,MAAI;GAEF,MAAM,OADS,MAAM,KAAK,SAAS,OAAO,WAAW,aAAa,CAAC,GAC/C,MAAM;AAC1B,OAAI,OAAO,QAAQ,YAAY,IAAI,WAAW,EAAG,QAAO;AACxD,OAAI;AACF,cAAU,KAAK,MAAM,IAAI;WACnB;AACN,cAAU;;UAEN;AAGN,OAAI,MAAM,KAAK,kBAAkB,aAAa,CAC5C,OAAM,IAAI,oBAAoB,aAAa;AAG7C,UAAO;;EAGT,MAAM,uBAAM,IAAI,MAAM,EAAC,aAAa;EACpC,MAAM,UAAoB,EAAE;AAE5B,MAAI,OAAO,UAAU,OAAO,YAAY,YAAY,YAAY,MAAM;GACpE,MAAM,cAAc,cAClB,SACA,OAAO,OACR;AACD,QAAK,MAAM,CAAC,WAAW,WAAW,OAAO,QAAQ,YAAY,CAC3D,MAAK,MAAM,SAAS,OAClB,KAAI,cAAc,OAChB,SAAQ,KAAK;IAAE,MAAM;IAAQ,YAAY;IAAK,WAAW;IAAO,CAAC;OAEjE,SAAQ,KAAK;IAAE,MAAM;IAAW,YAAY;IAAK;IAAO,CAAC;;EAMjE,MAAM,QAAsB;GAC1B;GACA,YAAY;GACZ,QAAQ,OAAO;GACf,OAAO;GACP,WAAW;GACX,WAAW;GACX,aAAa,QAAQ;GACtB;EACD,IAAI;AACJ,MAAI,OAAO,SAAS;AAClB,aAAU,mBAAmB,SAAS,OAAO,QAA6B;AAC1E,OAAI,OAAO,UAAU,OAAO,YAAY,YAAY,YAAY,KAC9D,WAAU,MAAM,uBAAuB;IACrC,KAAK,KAAK;IACV,UAAU;IACV,aAAa;IACb,QAAQ,OAAO;IACf,OAAO,OAAO;IACf,CAAC;;AAIN,MAAI;AACF,OAAI,KAAK,QAAQ,qBACf,OAAM,KAAK,QAAQ,qBAAqB;IACtC;IACA;IACA;IACD,CAAC;QACG;AACL,UAAM,KAAK,QAAQ,WAAW,MAAM;AACpC,UAAM,KAAK,QAAQ,aAAa,WAAW,QAAQ;AACnD,QAAI,QAAS,OAAM,KAAK,QAAQ,aAAa,WAAW,QAAQ;;WAE3D,KAAK;AACZ,SAAM,IAAI,oBACR,cACA,uBAAuB,eAAe,QAAQ,IAAI,UAAU,OAAO,IAAI,GACxE;;AAGH,MAAI,SAOF;OAAI,KAAK,SAAS,KAAK,QAAQ,eAC7B,KAAI;IACF,MAAM,SAAS,MAAM,KAAK,MAAM,QAAQ;AAIxC,QAAI,OAAQ,OAAM,KAAK,QAAQ,eAAe,WAAW,QAAQ,OAAO,KAAK;QACxE,KAAI,MAAM,0CAA0C,EAAE,WAAW,CAAC;YAChE,KAAK;AAKZ,QAAI,MAAM,8CAA8C;KACtD;KACA,OAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,IAAI;KACxD,CAAC;;;AAIR,SAAO;;;;;;CAOT,AAAQ,cAAc,QAA6B,cAA8B;AAE/E,SAAO,QADM,OAAO,SAAS,SACR,aAAa;;;AAItC,SAAS,UAAU,MAAc,SAA0B;CACzD,MAAM,WAAW,QACd,QAAQ,qBAAqB,OAAO,CACpC,QAAQ,SAAS,IAAI,CACrB,QAAQ,OAAO,QAAQ,CACvB,QAAQ,MAAM,KAAK,CACnB,QAAQ,OAAO,OAAO;AACzB,yBAAO,IAAI,OAAO,QAAQ,SAAS,GAAG,EAAC,KAAK,KAAK"}
|