@aigne/afs-index 1.12.0-beta.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE.md +26 -0
- package/dist/_virtual/_@oxc-project_runtime@0.108.0/helpers/decorate.cjs +11 -0
- package/dist/_virtual/_@oxc-project_runtime@0.108.0/helpers/decorate.mjs +10 -0
- package/dist/backend/d1-index-backend.cjs +656 -0
- package/dist/backend/d1-index-backend.d.cts +149 -0
- package/dist/backend/d1-index-backend.d.cts.map +1 -0
- package/dist/backend/d1-index-backend.d.mts +149 -0
- package/dist/backend/d1-index-backend.d.mts.map +1 -0
- package/dist/backend/d1-index-backend.mjs +657 -0
- package/dist/backend/d1-index-backend.mjs.map +1 -0
- package/dist/backend/d1-vector-backend.cjs +83 -0
- package/dist/backend/d1-vector-backend.d.cts +21 -0
- package/dist/backend/d1-vector-backend.d.cts.map +1 -0
- package/dist/backend/d1-vector-backend.d.mts +21 -0
- package/dist/backend/d1-vector-backend.d.mts.map +1 -0
- package/dist/backend/d1-vector-backend.mjs +84 -0
- package/dist/backend/d1-vector-backend.mjs.map +1 -0
- package/dist/backend/memory-backend.cjs +304 -0
- package/dist/backend/memory-backend.d.cts +66 -0
- package/dist/backend/memory-backend.d.cts.map +1 -0
- package/dist/backend/memory-backend.d.mts +66 -0
- package/dist/backend/memory-backend.d.mts.map +1 -0
- package/dist/backend/memory-backend.mjs +305 -0
- package/dist/backend/memory-backend.mjs.map +1 -0
- package/dist/backend/prefix-range.cjs +25 -0
- package/dist/backend/prefix-range.mjs +25 -0
- package/dist/backend/prefix-range.mjs.map +1 -0
- package/dist/backend/schema.cjs +187 -0
- package/dist/backend/schema.mjs +188 -0
- package/dist/backend/schema.mjs.map +1 -0
- package/dist/backend/sqlite-backend.cjs +889 -0
- package/dist/backend/sqlite-backend.d.cts +96 -0
- package/dist/backend/sqlite-backend.d.cts.map +1 -0
- package/dist/backend/sqlite-backend.d.mts +96 -0
- package/dist/backend/sqlite-backend.d.mts.map +1 -0
- package/dist/backend/sqlite-backend.mjs +890 -0
- package/dist/backend/sqlite-backend.mjs.map +1 -0
- package/dist/backend/types.d.cts +209 -0
- package/dist/backend/types.d.cts.map +1 -0
- package/dist/backend/types.d.mts +209 -0
- package/dist/backend/types.d.mts.map +1 -0
- package/dist/backend/upstash-vector-backend.cjs +98 -0
- package/dist/backend/upstash-vector-backend.d.cts +21 -0
- package/dist/backend/upstash-vector-backend.d.cts.map +1 -0
- package/dist/backend/upstash-vector-backend.d.mts +21 -0
- package/dist/backend/upstash-vector-backend.d.mts.map +1 -0
- package/dist/backend/upstash-vector-backend.mjs +98 -0
- package/dist/backend/upstash-vector-backend.mjs.map +1 -0
- package/dist/backend/vector-backend.d.cts +29 -0
- package/dist/backend/vector-backend.d.cts.map +1 -0
- package/dist/backend/vector-backend.d.mts +29 -0
- package/dist/backend/vector-backend.d.mts.map +1 -0
- package/dist/dirty-queue.d.cts +41 -0
- package/dist/dirty-queue.d.cts.map +1 -0
- package/dist/dirty-queue.d.mts +41 -0
- package/dist/dirty-queue.d.mts.map +1 -0
- package/dist/embed.cjs +45 -0
- package/dist/embed.d.cts +20 -0
- package/dist/embed.d.cts.map +1 -0
- package/dist/embed.d.mts +20 -0
- package/dist/embed.d.mts.map +1 -0
- package/dist/embed.mjs +46 -0
- package/dist/embed.mjs.map +1 -0
- package/dist/extraction/defaults.cjs +242 -0
- package/dist/extraction/defaults.d.cts +14 -0
- package/dist/extraction/defaults.d.cts.map +1 -0
- package/dist/extraction/defaults.d.mts +14 -0
- package/dist/extraction/defaults.d.mts.map +1 -0
- package/dist/extraction/defaults.mjs +242 -0
- package/dist/extraction/defaults.mjs.map +1 -0
- package/dist/extraction/field-map.cjs +162 -0
- package/dist/extraction/field-map.d.cts +70 -0
- package/dist/extraction/field-map.d.cts.map +1 -0
- package/dist/extraction/field-map.d.mts +70 -0
- package/dist/extraction/field-map.d.mts.map +1 -0
- package/dist/extraction/field-map.mjs +157 -0
- package/dist/extraction/field-map.mjs.map +1 -0
- package/dist/extraction/manager.cjs +74 -0
- package/dist/extraction/manager.d.cts +47 -0
- package/dist/extraction/manager.d.cts.map +1 -0
- package/dist/extraction/manager.d.mts +47 -0
- package/dist/extraction/manager.d.mts.map +1 -0
- package/dist/extraction/manager.mjs +74 -0
- package/dist/extraction/manager.mjs.map +1 -0
- package/dist/extraction/path-anchors.cjs +86 -0
- package/dist/extraction/path-anchors.d.cts +72 -0
- package/dist/extraction/path-anchors.d.cts.map +1 -0
- package/dist/extraction/path-anchors.d.mts +72 -0
- package/dist/extraction/path-anchors.d.mts.map +1 -0
- package/dist/extraction/path-anchors.mjs +86 -0
- package/dist/extraction/path-anchors.mjs.map +1 -0
- package/dist/follower-scheduler.cjs +92 -0
- package/dist/follower-scheduler.d.cts +58 -0
- package/dist/follower-scheduler.d.cts.map +1 -0
- package/dist/follower-scheduler.d.mts +58 -0
- package/dist/follower-scheduler.d.mts.map +1 -0
- package/dist/follower-scheduler.mjs +92 -0
- package/dist/follower-scheduler.mjs.map +1 -0
- package/dist/follower.cjs +577 -0
- package/dist/follower.d.cts +241 -0
- package/dist/follower.d.cts.map +1 -0
- package/dist/follower.d.mts +241 -0
- package/dist/follower.d.mts.map +1 -0
- package/dist/follower.mjs +571 -0
- package/dist/follower.mjs.map +1 -0
- package/dist/index-provider.cjs +1166 -0
- package/dist/index-provider.d.cts +175 -0
- package/dist/index-provider.d.cts.map +1 -0
- package/dist/index-provider.d.mts +175 -0
- package/dist/index-provider.d.mts.map +1 -0
- package/dist/index-provider.mjs +1167 -0
- package/dist/index-provider.mjs.map +1 -0
- package/dist/index.cjs +53 -0
- package/dist/index.d.cts +22 -0
- package/dist/index.d.mts +22 -0
- package/dist/index.mjs +20 -0
- package/dist/keys/exact.cjs +20 -0
- package/dist/keys/exact.mjs +20 -0
- package/dist/keys/exact.mjs.map +1 -0
- package/dist/keys/location.cjs +13 -0
- package/dist/keys/location.mjs +13 -0
- package/dist/keys/location.mjs.map +1 -0
- package/dist/keys/normalized.cjs +62 -0
- package/dist/keys/normalized.mjs +63 -0
- package/dist/keys/normalized.mjs.map +1 -0
- package/dist/keys/person.cjs +15 -0
- package/dist/keys/person.mjs +15 -0
- package/dist/keys/person.mjs.map +1 -0
- package/dist/keys/registry.cjs +45 -0
- package/dist/keys/registry.d.cts +15 -0
- package/dist/keys/registry.d.cts.map +1 -0
- package/dist/keys/registry.d.mts +15 -0
- package/dist/keys/registry.d.mts.map +1 -0
- package/dist/keys/registry.mjs +46 -0
- package/dist/keys/registry.mjs.map +1 -0
- package/dist/keys/time.cjs +128 -0
- package/dist/keys/time.mjs +128 -0
- package/dist/keys/time.mjs.map +1 -0
- package/dist/keys/topic.cjs +23 -0
- package/dist/keys/topic.mjs +23 -0
- package/dist/keys/topic.mjs.map +1 -0
- package/dist/manifest-index.cjs +29 -0
- package/dist/manifest-index.d.cts +24 -0
- package/dist/manifest-index.d.cts.map +1 -0
- package/dist/manifest-index.d.mts +24 -0
- package/dist/manifest-index.d.mts.map +1 -0
- package/dist/manifest-index.mjs +28 -0
- package/dist/manifest-index.mjs.map +1 -0
- package/dist/offline-rebuild.cjs +48 -0
- package/dist/offline-rebuild.d.cts +57 -0
- package/dist/offline-rebuild.d.cts.map +1 -0
- package/dist/offline-rebuild.d.mts +57 -0
- package/dist/offline-rebuild.d.mts.map +1 -0
- package/dist/offline-rebuild.mjs +49 -0
- package/dist/offline-rebuild.mjs.map +1 -0
- package/dist/query/engine.cjs +433 -0
- package/dist/query/engine.d.cts +34 -0
- package/dist/query/engine.d.cts.map +1 -0
- package/dist/query/engine.d.mts +34 -0
- package/dist/query/engine.d.mts.map +1 -0
- package/dist/query/engine.mjs +433 -0
- package/dist/query/engine.mjs.map +1 -0
- package/dist/query/limits.cjs +12 -0
- package/dist/query/limits.mjs +12 -0
- package/dist/query/limits.mjs.map +1 -0
- package/dist/text/cjk.cjs +30 -0
- package/dist/text/cjk.d.cts +24 -0
- package/dist/text/cjk.d.cts.map +1 -0
- package/dist/text/cjk.d.mts +24 -0
- package/dist/text/cjk.d.mts.map +1 -0
- package/dist/text/cjk.mjs +30 -0
- package/dist/text/cjk.mjs.map +1 -0
- package/dist/text/fts.cjs +244 -0
- package/dist/text/fts.mjs +245 -0
- package/dist/text/fts.mjs.map +1 -0
- package/dist/text/host.cjs +39 -0
- package/dist/text/host.mjs +38 -0
- package/dist/text/host.mjs.map +1 -0
- package/dist/types.cjs +44 -0
- package/dist/types.d.cts +393 -0
- package/dist/types.d.cts.map +1 -0
- package/dist/types.d.mts +393 -0
- package/dist/types.d.mts.map +1 -0
- package/dist/types.mjs +42 -0
- package/dist/types.mjs.map +1 -0
- package/package.json +59 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.mts","names":[],"sources":["../../src/backend/types.ts"],"mappings":";;;;UAoBiB,SAAA;EACf,UAAA;EACA,OAAA;EACA,UAAA,GAAa,MAAA;EACb,SAAA;AAAA;;;AAiBF;;;;;;;;;;;;UAAiB,uBAAA;EACf,KAAA;EACA,UAAA;EACA,WAAA;EAiBA;EAfA,WAAA;EAgBA;;;;EAXA,SAAA;EAyBe;;;;;EAnBf,QAAA;AAAA;AAAA,UAGe,oBAAA;EACf,KAAA,EAAO,YAAA;EACP,OAAA,GAAU,MAAA;EACV,OAAA;AAAA;;;;;;;;;;;UAae,kBAAA;EACf,UAAA;AAAA;AAAA,UAGe,YAAA;EA2CgC;EAzC/C,UAAA,IAAc,OAAA;EA0C4B;EAvC1C,KAAA,IAAS,OAAA;EAwCoB;;;;;;;;;;;;;;;EAvB7B,cAAA,EAAgB,QAAA,EAAU,oBAAA;EAG1B,UAAA,CAAW,KAAA,EAAO,YAAA,GAAe,OAAA;EA0E9B;;;;;;;EAlEH,oBAAA,EAAsB,QAAA,EAAU,oBAAA,GAAuB,OAAA;EACvD,QAAA,CAAS,IAAA,WAAe,OAAA,CAAQ,YAAA;EAChC,WAAA,CAAY,IAAA,WAAe,OAAA;EAmGW;;EAhGtC,qBAAA,EAAuB,IAAA,UAAc,IAAA,GAAO,kBAAA,GAAqB,OAAA;EACjE,WAAA,CAAY,MAAA,EAAQ,WAAA,GAAc,OAAA,CAAQ,YAAA;EAC1C,SAAA,CAAU,MAAA,EAAQ,WAAA,GAAc,OAAA;EAGhC,YAAA,CAAa,IAAA,UAAc,OAAA,EAAS,MAAA,KAAW,OAAA;EAC/C,YAAA,CAAa,KAAA,EAAO,WAAA,GAAc,OAAA,CAAQ,YAAA;EAC1C,aAAA,CAAc,IAAA,WAAe,OAAA;EAC7B,aAAA,CAAc,IAAA,WAAe,OAAA,CAAQ,UAAA;EAGrC,YAAA,CAAa,MAAA,EAAQ,WAAA,GAAc,OAAA;EAGnC,YAAA,CAAa,IAAA,UAAc,OAAA,WAAkB,OAAA;EA/CpC;;;;;;EAsDT,UAAA,EAAY,IAAA,WAAe,OAAA;EAlCM;;;;;;;;;EA4CjC,QAAA,CACE,IAAA,UACA,KAAA,WACA,UAAA,GAAa,MAAA,mBACb,IAAA;IAAS,MAAA;IAAiB,OAAA;IAAoB,KAAA;IAAgB,UAAA;EAAA,IAC7D,OAAA,CAAQ,SAAA;EACX,aAAA,CAAc,IAAA,WAAe,OAAA;EAG7B,aAAA,EACE,IAAA;IAAQ,IAAA;IAAc,MAAA;EAAA,KACtB,IAAA,WACA,IAAA,GAAO,SAAA,GACN,OAAA,CAAQ,cAAA;EAGX,UAAA,EAAY,UAAA,WAAqB,OAAA,CAAQ,SAAA;EA9CC;;;;;;;;;EAwD1C,WAAA,EACE,UAAA,UACA,OAAA,UACA,UAAA,GAAa,MAAA,mBACZ,OAAA;EAvDH;;;;;;EA8DA,gBAAA,EAAkB,UAAA,WAAqB,OAAA;EA7DV;;;;;;EAoE7B,UAAA,EACE,UAAA,UACA,YAAA;IAAiB,WAAA;IAAsB,WAAA;EAAA,IACtC,OAAA;EAhEU;;;;;;;;;;;EA4Eb,gBAAA,EAAkB,UAAA,WAAqB,OAAA,CAAQ,uBAAA;EAG/C,iBAAA,EAAmB,OAAA,EAAS,YAAA,KAAiB,OAAA;EAC7C,iBAAA,EAAmB,KAAA;IAAS,IAAA;IAAc,OAAA,EAAS,MAAA;EAAA,MAAe,OAAA;EAClE,mBAAA,EAAqB,KAAA;IAAS,IAAA;IAAc,OAAA;EAAA,MAAsB,OAAA;EAClE,YAAA,EAAc,KAAA,YAAiB,IAAA,GAAO,kBAAA,GAAqB,OAAA;EAvDjD;;;;;;;EAiEV,cAAA,EAAgB,IAAA,UAAc,SAAA,YAAqB,MAAA,YAAkB,OAAA;EA3DrE;;;;;;EAkEA,WAAA,EACE,SAAA,YACA,KAAA,WACA,IAAA;IAAS,MAAA;IAAiB,OAAA;IAAoB,KAAA;IAAgB,UAAA;EAAA,IAC7D,OAAA,CAAQ,YAAA;EACX,eAAA,EAAiB,IAAA,WAAe,OAAA;EAChC,mBAAA;IAA0B,SAAA;IAAoB,SAAA;EAAA;AAAA"}
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
|
|
2
|
+
//#region src/backend/upstash-vector-backend.ts
|
|
3
|
+
const UPSTASH_MAX_TOP_K = 1e3;
|
|
4
|
+
var UpstashVectorBackend = class {
|
|
5
|
+
constructor(url, token, scope) {
|
|
6
|
+
this.url = url;
|
|
7
|
+
this.token = token;
|
|
8
|
+
this.scope = scope;
|
|
9
|
+
}
|
|
10
|
+
async upsert(entryPath, vector, domain) {
|
|
11
|
+
const id = await hashVectorId(this.scope, entryPath);
|
|
12
|
+
const metadata = {
|
|
13
|
+
scope: this.scope,
|
|
14
|
+
entryPath
|
|
15
|
+
};
|
|
16
|
+
if (domain) metadata.domain = domain;
|
|
17
|
+
await this.request("upsert", [{
|
|
18
|
+
id,
|
|
19
|
+
vector,
|
|
20
|
+
metadata
|
|
21
|
+
}]);
|
|
22
|
+
}
|
|
23
|
+
async query(vector, filter, topK = 20) {
|
|
24
|
+
const boundedTopK = Math.min(topK, UPSTASH_MAX_TOP_K);
|
|
25
|
+
const filterStr = buildUpstashFilter(this.scope, filter);
|
|
26
|
+
const matches = await this.request("query", {
|
|
27
|
+
vector,
|
|
28
|
+
topK: boundedTopK,
|
|
29
|
+
filter: filterStr,
|
|
30
|
+
includeMetadata: true
|
|
31
|
+
});
|
|
32
|
+
if (!Array.isArray(matches)) return [];
|
|
33
|
+
let results = matches.filter((m) => typeof m.metadata?.entryPath === "string").map((m) => ({
|
|
34
|
+
entryPath: m.metadata.entryPath,
|
|
35
|
+
similarity: m.score
|
|
36
|
+
}));
|
|
37
|
+
if (filter.pathPrefix) {
|
|
38
|
+
const prefix = filter.pathPrefix;
|
|
39
|
+
results = results.filter((r) => r.entryPath.startsWith(prefix));
|
|
40
|
+
}
|
|
41
|
+
return results;
|
|
42
|
+
}
|
|
43
|
+
async delete(entryPath) {
|
|
44
|
+
const id = await hashVectorId(this.scope, entryPath);
|
|
45
|
+
await this.request("delete", [id]);
|
|
46
|
+
}
|
|
47
|
+
capability() {
|
|
48
|
+
return {
|
|
49
|
+
supported: true,
|
|
50
|
+
dimension: 0
|
|
51
|
+
};
|
|
52
|
+
}
|
|
53
|
+
async request(endpoint, body) {
|
|
54
|
+
const res = await fetch(`${this.url}/${endpoint}`, {
|
|
55
|
+
method: "POST",
|
|
56
|
+
headers: {
|
|
57
|
+
Authorization: `Bearer ${this.token}`,
|
|
58
|
+
"Content-Type": "application/json"
|
|
59
|
+
},
|
|
60
|
+
body: JSON.stringify(body)
|
|
61
|
+
});
|
|
62
|
+
if (!res.ok) {
|
|
63
|
+
const text = await res.text().catch(() => "(unreadable)");
|
|
64
|
+
throw new Error(`Upstash Vector /${endpoint}: ${res.status} ${text}`);
|
|
65
|
+
}
|
|
66
|
+
return (await res.json()).result;
|
|
67
|
+
}
|
|
68
|
+
};
|
|
69
|
+
/**
|
|
70
|
+
* Build an Upstash SQL-like filter string for the query.
|
|
71
|
+
* Scope is always required (enforces per-user isolation).
|
|
72
|
+
* Domain values are escaped to prevent SQL injection via single-quote doubling.
|
|
73
|
+
*/
|
|
74
|
+
function buildUpstashFilter(scope, filter) {
|
|
75
|
+
const clauses = [`scope = '${escapeSql(scope)}'`];
|
|
76
|
+
if (filter.domain) clauses.push(`domain = '${escapeSql(filter.domain)}'`);
|
|
77
|
+
else if (filter.domains && filter.domains.length > 0) {
|
|
78
|
+
const list = filter.domains.map((d) => `'${escapeSql(d)}'`).join(", ");
|
|
79
|
+
clauses.push(`domain IN (${list})`);
|
|
80
|
+
}
|
|
81
|
+
return clauses.join(" AND ");
|
|
82
|
+
}
|
|
83
|
+
/** Escape a string for embedding in a SQL-like Upstash filter (double single-quotes). */
|
|
84
|
+
function escapeSql(s) {
|
|
85
|
+
return s.replace(/'/g, "''");
|
|
86
|
+
}
|
|
87
|
+
/**
|
|
88
|
+
* Derive a stable vector ID from scope + entryPath.
|
|
89
|
+
* SHA-256 → 64 hex chars, which is safely within any vector store ID limit.
|
|
90
|
+
*/
|
|
91
|
+
async function hashVectorId(scope, entryPath) {
|
|
92
|
+
const data = new TextEncoder().encode(`${scope}:${entryPath}`);
|
|
93
|
+
const buf = await crypto.subtle.digest("SHA-256", data);
|
|
94
|
+
return Array.from(new Uint8Array(buf)).map((b) => b.toString(16).padStart(2, "0")).join("");
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
//#endregion
|
|
98
|
+
exports.UpstashVectorBackend = UpstashVectorBackend;
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { VectorResult } from "../types.cjs";
|
|
2
|
+
import { VectorBackend, VectorFilter } from "./vector-backend.cjs";
|
|
3
|
+
|
|
4
|
+
//#region src/backend/upstash-vector-backend.d.ts
|
|
5
|
+
declare class UpstashVectorBackend implements VectorBackend {
|
|
6
|
+
private readonly url;
|
|
7
|
+
private readonly token;
|
|
8
|
+
private readonly scope;
|
|
9
|
+
constructor(url: string, token: string, scope: string);
|
|
10
|
+
upsert(entryPath: string, vector: number[], domain?: string): Promise<void>;
|
|
11
|
+
query(vector: number[], filter: VectorFilter, topK?: number): Promise<VectorResult[]>;
|
|
12
|
+
delete(entryPath: string): Promise<void>;
|
|
13
|
+
capability(): {
|
|
14
|
+
supported: boolean;
|
|
15
|
+
dimension: number;
|
|
16
|
+
} | null;
|
|
17
|
+
private request;
|
|
18
|
+
}
|
|
19
|
+
//#endregion
|
|
20
|
+
export { UpstashVectorBackend };
|
|
21
|
+
//# sourceMappingURL=upstash-vector-backend.d.cts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"upstash-vector-backend.d.cts","names":[],"sources":["../../src/backend/upstash-vector-backend.ts"],"mappings":";;;;cA6Ba,oBAAA,YAAgC,aAAA;EAAA,iBAExB,GAAA;EAAA,iBACA,KAAA;EAAA,iBACA,KAAA;cAFA,GAAA,UACA,KAAA,UACA,KAAA;EAGb,MAAA,CAAO,SAAA,UAAmB,MAAA,YAAkB,MAAA,YAAkB,OAAA;EAS9D,KAAA,CAAM,MAAA,YAAkB,MAAA,EAAQ,YAAA,EAAc,IAAA,YAAY,OAAA,CAAQ,YAAA;EAoBlE,MAAA,CAAO,SAAA,WAAoB,OAAA;EAKjC,UAAA,CAAA;IAAgB,SAAA;IAAoB,SAAA;EAAA;EAAA,QAItB,OAAA;AAAA"}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { VectorResult } from "../types.mjs";
|
|
2
|
+
import { VectorBackend, VectorFilter } from "./vector-backend.mjs";
|
|
3
|
+
|
|
4
|
+
//#region src/backend/upstash-vector-backend.d.ts
|
|
5
|
+
declare class UpstashVectorBackend implements VectorBackend {
|
|
6
|
+
private readonly url;
|
|
7
|
+
private readonly token;
|
|
8
|
+
private readonly scope;
|
|
9
|
+
constructor(url: string, token: string, scope: string);
|
|
10
|
+
upsert(entryPath: string, vector: number[], domain?: string): Promise<void>;
|
|
11
|
+
query(vector: number[], filter: VectorFilter, topK?: number): Promise<VectorResult[]>;
|
|
12
|
+
delete(entryPath: string): Promise<void>;
|
|
13
|
+
capability(): {
|
|
14
|
+
supported: boolean;
|
|
15
|
+
dimension: number;
|
|
16
|
+
} | null;
|
|
17
|
+
private request;
|
|
18
|
+
}
|
|
19
|
+
//#endregion
|
|
20
|
+
export { UpstashVectorBackend };
|
|
21
|
+
//# sourceMappingURL=upstash-vector-backend.d.mts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"upstash-vector-backend.d.mts","names":[],"sources":["../../src/backend/upstash-vector-backend.ts"],"mappings":";;;;cA6Ba,oBAAA,YAAgC,aAAA;EAAA,iBAExB,GAAA;EAAA,iBACA,KAAA;EAAA,iBACA,KAAA;cAFA,GAAA,UACA,KAAA,UACA,KAAA;EAGb,MAAA,CAAO,SAAA,UAAmB,MAAA,YAAkB,MAAA,YAAkB,OAAA;EAS9D,KAAA,CAAM,MAAA,YAAkB,MAAA,EAAQ,YAAA,EAAc,IAAA,YAAY,OAAA,CAAQ,YAAA;EAoBlE,MAAA,CAAO,SAAA,WAAoB,OAAA;EAKjC,UAAA,CAAA;IAAgB,SAAA;IAAoB,SAAA;EAAA;EAAA,QAItB,OAAA;AAAA"}
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
//#region src/backend/upstash-vector-backend.ts
|
|
2
|
+
const UPSTASH_MAX_TOP_K = 1e3;
|
|
3
|
+
var UpstashVectorBackend = class {
|
|
4
|
+
constructor(url, token, scope) {
|
|
5
|
+
this.url = url;
|
|
6
|
+
this.token = token;
|
|
7
|
+
this.scope = scope;
|
|
8
|
+
}
|
|
9
|
+
async upsert(entryPath, vector, domain) {
|
|
10
|
+
const id = await hashVectorId(this.scope, entryPath);
|
|
11
|
+
const metadata = {
|
|
12
|
+
scope: this.scope,
|
|
13
|
+
entryPath
|
|
14
|
+
};
|
|
15
|
+
if (domain) metadata.domain = domain;
|
|
16
|
+
await this.request("upsert", [{
|
|
17
|
+
id,
|
|
18
|
+
vector,
|
|
19
|
+
metadata
|
|
20
|
+
}]);
|
|
21
|
+
}
|
|
22
|
+
async query(vector, filter, topK = 20) {
|
|
23
|
+
const boundedTopK = Math.min(topK, UPSTASH_MAX_TOP_K);
|
|
24
|
+
const filterStr = buildUpstashFilter(this.scope, filter);
|
|
25
|
+
const matches = await this.request("query", {
|
|
26
|
+
vector,
|
|
27
|
+
topK: boundedTopK,
|
|
28
|
+
filter: filterStr,
|
|
29
|
+
includeMetadata: true
|
|
30
|
+
});
|
|
31
|
+
if (!Array.isArray(matches)) return [];
|
|
32
|
+
let results = matches.filter((m) => typeof m.metadata?.entryPath === "string").map((m) => ({
|
|
33
|
+
entryPath: m.metadata.entryPath,
|
|
34
|
+
similarity: m.score
|
|
35
|
+
}));
|
|
36
|
+
if (filter.pathPrefix) {
|
|
37
|
+
const prefix = filter.pathPrefix;
|
|
38
|
+
results = results.filter((r) => r.entryPath.startsWith(prefix));
|
|
39
|
+
}
|
|
40
|
+
return results;
|
|
41
|
+
}
|
|
42
|
+
async delete(entryPath) {
|
|
43
|
+
const id = await hashVectorId(this.scope, entryPath);
|
|
44
|
+
await this.request("delete", [id]);
|
|
45
|
+
}
|
|
46
|
+
capability() {
|
|
47
|
+
return {
|
|
48
|
+
supported: true,
|
|
49
|
+
dimension: 0
|
|
50
|
+
};
|
|
51
|
+
}
|
|
52
|
+
async request(endpoint, body) {
|
|
53
|
+
const res = await fetch(`${this.url}/${endpoint}`, {
|
|
54
|
+
method: "POST",
|
|
55
|
+
headers: {
|
|
56
|
+
Authorization: `Bearer ${this.token}`,
|
|
57
|
+
"Content-Type": "application/json"
|
|
58
|
+
},
|
|
59
|
+
body: JSON.stringify(body)
|
|
60
|
+
});
|
|
61
|
+
if (!res.ok) {
|
|
62
|
+
const text = await res.text().catch(() => "(unreadable)");
|
|
63
|
+
throw new Error(`Upstash Vector /${endpoint}: ${res.status} ${text}`);
|
|
64
|
+
}
|
|
65
|
+
return (await res.json()).result;
|
|
66
|
+
}
|
|
67
|
+
};
|
|
68
|
+
/**
|
|
69
|
+
* Build an Upstash SQL-like filter string for the query.
|
|
70
|
+
* Scope is always required (enforces per-user isolation).
|
|
71
|
+
* Domain values are escaped to prevent SQL injection via single-quote doubling.
|
|
72
|
+
*/
|
|
73
|
+
function buildUpstashFilter(scope, filter) {
|
|
74
|
+
const clauses = [`scope = '${escapeSql(scope)}'`];
|
|
75
|
+
if (filter.domain) clauses.push(`domain = '${escapeSql(filter.domain)}'`);
|
|
76
|
+
else if (filter.domains && filter.domains.length > 0) {
|
|
77
|
+
const list = filter.domains.map((d) => `'${escapeSql(d)}'`).join(", ");
|
|
78
|
+
clauses.push(`domain IN (${list})`);
|
|
79
|
+
}
|
|
80
|
+
return clauses.join(" AND ");
|
|
81
|
+
}
|
|
82
|
+
/** Escape a string for embedding in a SQL-like Upstash filter (double single-quotes). */
|
|
83
|
+
function escapeSql(s) {
|
|
84
|
+
return s.replace(/'/g, "''");
|
|
85
|
+
}
|
|
86
|
+
/**
|
|
87
|
+
* Derive a stable vector ID from scope + entryPath.
|
|
88
|
+
* SHA-256 → 64 hex chars, which is safely within any vector store ID limit.
|
|
89
|
+
*/
|
|
90
|
+
async function hashVectorId(scope, entryPath) {
|
|
91
|
+
const data = new TextEncoder().encode(`${scope}:${entryPath}`);
|
|
92
|
+
const buf = await crypto.subtle.digest("SHA-256", data);
|
|
93
|
+
return Array.from(new Uint8Array(buf)).map((b) => b.toString(16).padStart(2, "0")).join("");
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
//#endregion
|
|
97
|
+
export { UpstashVectorBackend };
|
|
98
|
+
//# sourceMappingURL=upstash-vector-backend.mjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"upstash-vector-backend.mjs","names":[],"sources":["../../src/backend/upstash-vector-backend.ts"],"sourcesContent":["/**\n * UpstashVectorBackend — VectorBackend backed by Upstash Vector REST API.\n *\n * Upstash Vector uses HTTP (no TCP), making it compatible with Cloudflare\n * Workers. Each instance is bound to one scope; isolation is enforced via\n * a metadata field filter on every query.\n *\n * Platform limits to be aware of:\n * - Free plan: 10,000 vectors, 1,536 dimensions\n * - Metadata filter syntax: SQL-like (`field = 'val' AND ...`)\n * - topK: up to 1,000 per query\n * - No vector ID length limit documented; SHA-256 hex (64 chars) is safe\n *\n * Env vars (read by the CF runtime factory, not here):\n * AFS_UPSTASH_VECTOR_URL — Upstash REST endpoint (e.g. https://xxx.upstash.io)\n * AFS_UPSTASH_VECTOR_TOKEN — read-write token (secret)\n */\nimport type { VectorResult } from \"../types.js\";\nimport type { VectorBackend, VectorFilter } from \"./vector-backend.js\";\n\ninterface UpstashQueryMatch {\n id: string;\n score: number;\n metadata?: Record<string, unknown>;\n}\n\n// Max topK Upstash Vector supports per query.\nconst UPSTASH_MAX_TOP_K = 1000;\n\nexport class UpstashVectorBackend implements VectorBackend {\n constructor(\n private readonly url: string,\n private readonly token: string,\n private readonly scope: string,\n ) {}\n\n async upsert(entryPath: string, vector: number[], domain?: string): Promise<void> {\n const id = await hashVectorId(this.scope, entryPath);\n // `domain` is filtered on at query time (buildUpstashFilter) — it MUST be\n // stored here, else a boundDomains query's `domain IN (...)` matches nothing.\n const metadata: Record<string, unknown> = { scope: this.scope, entryPath };\n if (domain) metadata.domain = domain;\n await this.request(\"upsert\", [{ id, vector, metadata }]);\n }\n\n async query(vector: number[], filter: VectorFilter, topK = 20): Promise<VectorResult[]> {\n const boundedTopK = Math.min(topK, UPSTASH_MAX_TOP_K);\n const filterStr = buildUpstashFilter(this.scope, filter);\n const matches = await this.request<UpstashQueryMatch[]>(\"query\", {\n vector,\n topK: boundedTopK,\n filter: filterStr,\n includeMetadata: true,\n });\n if (!Array.isArray(matches)) return [];\n let results: VectorResult[] = matches\n .filter((m) => typeof m.metadata?.entryPath === \"string\")\n .map((m) => ({ entryPath: m.metadata!.entryPath as string, similarity: m.score }));\n if (filter.pathPrefix) {\n const prefix = filter.pathPrefix;\n results = results.filter((r) => r.entryPath.startsWith(prefix));\n }\n return results;\n }\n\n async delete(entryPath: string): Promise<void> {\n const id = await hashVectorId(this.scope, entryPath);\n await this.request(\"delete\", [id]);\n }\n\n capability(): { supported: boolean; dimension: number } | null {\n return { supported: true, dimension: 0 };\n }\n\n private async request<T = unknown>(endpoint: string, body: unknown): Promise<T> {\n const res = await fetch(`${this.url}/${endpoint}`, {\n method: \"POST\",\n headers: {\n Authorization: `Bearer ${this.token}`,\n \"Content-Type\": \"application/json\",\n },\n body: JSON.stringify(body),\n });\n if (!res.ok) {\n const text = await res.text().catch(() => \"(unreadable)\");\n throw new Error(`Upstash Vector /${endpoint}: ${res.status} ${text}`);\n }\n const json = (await res.json()) as { result: T };\n return json.result;\n }\n}\n\n/**\n * Build an Upstash SQL-like filter string for the query.\n * Scope is always required (enforces per-user isolation).\n * Domain values are escaped to prevent SQL injection via single-quote doubling.\n */\nfunction buildUpstashFilter(scope: string, filter: VectorFilter): string {\n const clauses: string[] = [`scope = '${escapeSql(scope)}'`];\n if (filter.domain) {\n clauses.push(`domain = '${escapeSql(filter.domain)}'`);\n } else if (filter.domains && filter.domains.length > 0) {\n const list = filter.domains.map((d) => `'${escapeSql(d)}'`).join(\", \");\n clauses.push(`domain IN (${list})`);\n }\n return clauses.join(\" AND \");\n}\n\n/** Escape a string for embedding in a SQL-like Upstash filter (double single-quotes). */\nfunction escapeSql(s: string): string {\n return s.replace(/'/g, \"''\");\n}\n\n/**\n * Derive a stable vector ID from scope + entryPath.\n * SHA-256 → 64 hex chars, which is safely within any vector store ID limit.\n */\nasync function hashVectorId(scope: string, entryPath: string): Promise<string> {\n const data = new TextEncoder().encode(`${scope}:${entryPath}`);\n const buf = await crypto.subtle.digest(\"SHA-256\", data);\n return Array.from(new Uint8Array(buf))\n .map((b) => b.toString(16).padStart(2, \"0\"))\n .join(\"\");\n}\n"],"mappings":";AA2BA,MAAM,oBAAoB;AAE1B,IAAa,uBAAb,MAA2D;CACzD,YACE,AAAiB,KACjB,AAAiB,OACjB,AAAiB,OACjB;EAHiB;EACA;EACA;;CAGnB,MAAM,OAAO,WAAmB,QAAkB,QAAgC;EAChF,MAAM,KAAK,MAAM,aAAa,KAAK,OAAO,UAAU;EAGpD,MAAM,WAAoC;GAAE,OAAO,KAAK;GAAO;GAAW;AAC1E,MAAI,OAAQ,UAAS,SAAS;AAC9B,QAAM,KAAK,QAAQ,UAAU,CAAC;GAAE;GAAI;GAAQ;GAAU,CAAC,CAAC;;CAG1D,MAAM,MAAM,QAAkB,QAAsB,OAAO,IAA6B;EACtF,MAAM,cAAc,KAAK,IAAI,MAAM,kBAAkB;EACrD,MAAM,YAAY,mBAAmB,KAAK,OAAO,OAAO;EACxD,MAAM,UAAU,MAAM,KAAK,QAA6B,SAAS;GAC/D;GACA,MAAM;GACN,QAAQ;GACR,iBAAiB;GAClB,CAAC;AACF,MAAI,CAAC,MAAM,QAAQ,QAAQ,CAAE,QAAO,EAAE;EACtC,IAAI,UAA0B,QAC3B,QAAQ,MAAM,OAAO,EAAE,UAAU,cAAc,SAAS,CACxD,KAAK,OAAO;GAAE,WAAW,EAAE,SAAU;GAAqB,YAAY,EAAE;GAAO,EAAE;AACpF,MAAI,OAAO,YAAY;GACrB,MAAM,SAAS,OAAO;AACtB,aAAU,QAAQ,QAAQ,MAAM,EAAE,UAAU,WAAW,OAAO,CAAC;;AAEjE,SAAO;;CAGT,MAAM,OAAO,WAAkC;EAC7C,MAAM,KAAK,MAAM,aAAa,KAAK,OAAO,UAAU;AACpD,QAAM,KAAK,QAAQ,UAAU,CAAC,GAAG,CAAC;;CAGpC,aAA+D;AAC7D,SAAO;GAAE,WAAW;GAAM,WAAW;GAAG;;CAG1C,MAAc,QAAqB,UAAkB,MAA2B;EAC9E,MAAM,MAAM,MAAM,MAAM,GAAG,KAAK,IAAI,GAAG,YAAY;GACjD,QAAQ;GACR,SAAS;IACP,eAAe,UAAU,KAAK;IAC9B,gBAAgB;IACjB;GACD,MAAM,KAAK,UAAU,KAAK;GAC3B,CAAC;AACF,MAAI,CAAC,IAAI,IAAI;GACX,MAAM,OAAO,MAAM,IAAI,MAAM,CAAC,YAAY,eAAe;AACzD,SAAM,IAAI,MAAM,mBAAmB,SAAS,IAAI,IAAI,OAAO,GAAG,OAAO;;AAGvE,UADc,MAAM,IAAI,MAAM,EAClB;;;;;;;;AAShB,SAAS,mBAAmB,OAAe,QAA8B;CACvE,MAAM,UAAoB,CAAC,YAAY,UAAU,MAAM,CAAC,GAAG;AAC3D,KAAI,OAAO,OACT,SAAQ,KAAK,aAAa,UAAU,OAAO,OAAO,CAAC,GAAG;UAC7C,OAAO,WAAW,OAAO,QAAQ,SAAS,GAAG;EACtD,MAAM,OAAO,OAAO,QAAQ,KAAK,MAAM,IAAI,UAAU,EAAE,CAAC,GAAG,CAAC,KAAK,KAAK;AACtE,UAAQ,KAAK,cAAc,KAAK,GAAG;;AAErC,QAAO,QAAQ,KAAK,QAAQ;;;AAI9B,SAAS,UAAU,GAAmB;AACpC,QAAO,EAAE,QAAQ,MAAM,KAAK;;;;;;AAO9B,eAAe,aAAa,OAAe,WAAoC;CAC7E,MAAM,OAAO,IAAI,aAAa,CAAC,OAAO,GAAG,MAAM,GAAG,YAAY;CAC9D,MAAM,MAAM,MAAM,OAAO,OAAO,OAAO,WAAW,KAAK;AACvD,QAAO,MAAM,KAAK,IAAI,WAAW,IAAI,CAAC,CACnC,KAAK,MAAM,EAAE,SAAS,GAAG,CAAC,SAAS,GAAG,IAAI,CAAC,CAC3C,KAAK,GAAG"}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import { VectorResult } from "../types.cjs";
|
|
2
|
+
|
|
3
|
+
//#region src/backend/vector-backend.d.ts
|
|
4
|
+
interface VectorFilter {
|
|
5
|
+
domain?: string;
|
|
6
|
+
domains?: string[];
|
|
7
|
+
pathPrefix?: string;
|
|
8
|
+
}
|
|
9
|
+
interface VectorBackend {
|
|
10
|
+
/**
|
|
11
|
+
* Store a vector for `entryPath`. `domain` (the entry's composite
|
|
12
|
+
* `{name}:{scope}`) MUST be persisted alongside so `query`'s `VectorFilter.
|
|
13
|
+
* domain(s)` can isolate cross-blocklet: backends without a join to
|
|
14
|
+
* `ds_indexed_entry` (Upstash/Vectorize) rely on it being in their own
|
|
15
|
+
* metadata — the D1 backend, which joins, ignores it. Omitting it makes the
|
|
16
|
+
* domain filter match nothing (empty results for any caller that passes
|
|
17
|
+
* boundDomains — i.e. every /user query).
|
|
18
|
+
*/
|
|
19
|
+
upsert(entryPath: string, vector: number[], domain?: string): Promise<void>;
|
|
20
|
+
query(vector: number[], filter: VectorFilter, topK?: number): Promise<VectorResult[]>;
|
|
21
|
+
delete(entryPath: string): Promise<void>;
|
|
22
|
+
capability(): {
|
|
23
|
+
supported: boolean;
|
|
24
|
+
dimension: number;
|
|
25
|
+
} | null;
|
|
26
|
+
}
|
|
27
|
+
//#endregion
|
|
28
|
+
export { VectorBackend, VectorFilter };
|
|
29
|
+
//# sourceMappingURL=vector-backend.d.cts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"vector-backend.d.cts","names":[],"sources":["../../src/backend/vector-backend.ts"],"mappings":";;;UASiB,YAAA;EACf,MAAA;EACA,OAAA;EACA,UAAA;AAAA;AAAA,UAGe,aAAA;;;;;;;;;;EAUf,MAAA,CAAO,SAAA,UAAmB,MAAA,YAAkB,MAAA,YAAkB,OAAA;EAC9D,KAAA,CAAM,MAAA,YAAkB,MAAA,EAAQ,YAAA,EAAc,IAAA,YAAgB,OAAA,CAAQ,YAAA;EACtE,MAAA,CAAO,SAAA,WAAoB,OAAA;EAC3B,UAAA;IAAgB,SAAA;IAAoB,SAAA;EAAA;AAAA"}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import { VectorResult } from "../types.mjs";
|
|
2
|
+
|
|
3
|
+
//#region src/backend/vector-backend.d.ts
|
|
4
|
+
interface VectorFilter {
|
|
5
|
+
domain?: string;
|
|
6
|
+
domains?: string[];
|
|
7
|
+
pathPrefix?: string;
|
|
8
|
+
}
|
|
9
|
+
interface VectorBackend {
|
|
10
|
+
/**
|
|
11
|
+
* Store a vector for `entryPath`. `domain` (the entry's composite
|
|
12
|
+
* `{name}:{scope}`) MUST be persisted alongside so `query`'s `VectorFilter.
|
|
13
|
+
* domain(s)` can isolate cross-blocklet: backends without a join to
|
|
14
|
+
* `ds_indexed_entry` (Upstash/Vectorize) rely on it being in their own
|
|
15
|
+
* metadata — the D1 backend, which joins, ignores it. Omitting it makes the
|
|
16
|
+
* domain filter match nothing (empty results for any caller that passes
|
|
17
|
+
* boundDomains — i.e. every /user query).
|
|
18
|
+
*/
|
|
19
|
+
upsert(entryPath: string, vector: number[], domain?: string): Promise<void>;
|
|
20
|
+
query(vector: number[], filter: VectorFilter, topK?: number): Promise<VectorResult[]>;
|
|
21
|
+
delete(entryPath: string): Promise<void>;
|
|
22
|
+
capability(): {
|
|
23
|
+
supported: boolean;
|
|
24
|
+
dimension: number;
|
|
25
|
+
} | null;
|
|
26
|
+
}
|
|
27
|
+
//#endregion
|
|
28
|
+
export { VectorBackend, VectorFilter };
|
|
29
|
+
//# sourceMappingURL=vector-backend.d.mts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"vector-backend.d.mts","names":[],"sources":["../../src/backend/vector-backend.ts"],"mappings":";;;UASiB,YAAA;EACf,MAAA;EACA,OAAA;EACA,UAAA;AAAA;AAAA,UAGe,aAAA;;;;;;;;;;EAUf,MAAA,CAAO,SAAA,UAAmB,MAAA,YAAkB,MAAA,YAAkB,OAAA;EAC9D,KAAA,CAAM,MAAA,YAAkB,MAAA,EAAQ,YAAA,EAAc,IAAA,YAAgB,OAAA,CAAQ,YAAA;EACtE,MAAA,CAAO,SAAA,WAAoB,OAAA;EAC3B,UAAA;IAAgB,SAAA;IAAoB,SAAA;EAAA;AAAA"}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
//#region src/dirty-queue.d.ts
|
|
2
|
+
/**
|
|
3
|
+
* DirtyPathQueue — consumer-side contract the IndexFollower drains
|
|
4
|
+
* (planning/index-provider/dirty-path-follower, design §7).
|
|
5
|
+
*
|
|
6
|
+
* This is the STRUCTURAL mirror of `@aigne/afs-did-space`'s dirty-path-queue
|
|
7
|
+
* contract. The index package and the did-space package do NOT depend on each
|
|
8
|
+
* other (they meet only at the runtime wiring layer, CF/Node), so — exactly like
|
|
9
|
+
* `spaceAfs: AFSRoot` and the `IndexBackend` interface — the follower declares
|
|
10
|
+
* the shape it needs and the runtime passes a did-space `PrefixedTreeIndex`
|
|
11
|
+
* (which implements these three methods) as a structurally-compatible value.
|
|
12
|
+
* Keep this in lockstep with did-space `src/dirty-path-queue.ts`.
|
|
13
|
+
*/
|
|
14
|
+
type DirtyPathOp = "put" | "delete";
|
|
15
|
+
/**
|
|
16
|
+
* A dirty row handed to the follower. `path` is the fragment-STRIPPED
|
|
17
|
+
* `originalPath` (e.g. `items/x.json`) when the queue is a prefixed source
|
|
18
|
+
* (the normal wiring). `op/cid/latestSeq` drive scheduling + the ack race guard
|
|
19
|
+
* only — the follower re-reads live state before writing the index (design §4.3).
|
|
20
|
+
*/
|
|
21
|
+
interface DirtyPathRow {
|
|
22
|
+
path: string;
|
|
23
|
+
latestSeq: number;
|
|
24
|
+
op: DirtyPathOp;
|
|
25
|
+
cid?: string | null;
|
|
26
|
+
mtime: number;
|
|
27
|
+
attempts: number;
|
|
28
|
+
}
|
|
29
|
+
interface ListDirtyPathOptions {
|
|
30
|
+
prefix: string;
|
|
31
|
+
limit: number;
|
|
32
|
+
now?: number;
|
|
33
|
+
}
|
|
34
|
+
interface DirtyPathQueue {
|
|
35
|
+
listDirtyPaths(options: ListDirtyPathOptions): Promise<DirtyPathRow[]>;
|
|
36
|
+
ackDirtyPath(path: string, latestSeq: number): Promise<boolean>;
|
|
37
|
+
markDirtyPathError(path: string, latestSeq: number, error: string, nextAttemptAt: number): Promise<boolean>;
|
|
38
|
+
}
|
|
39
|
+
//#endregion
|
|
40
|
+
export { DirtyPathOp, DirtyPathQueue, DirtyPathRow, ListDirtyPathOptions };
|
|
41
|
+
//# sourceMappingURL=dirty-queue.d.cts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"dirty-queue.d.cts","names":[],"sources":["../src/dirty-queue.ts"],"mappings":";;AAaA;;;;;AAQA;;;;;;KARY,WAAA;;;;;;;UAQK,YAAA;EACf,IAAA;EACA,SAAA;EACA,EAAA,EAAI,WAAA;EACJ,GAAA;EACA,KAAA;EACA,QAAA;AAAA;AAAA,UAGe,oBAAA;EACf,MAAA;EACA,KAAA;EACA,GAAA;AAAA;AAAA,UAGe,cAAA;EACf,cAAA,CAAe,OAAA,EAAS,oBAAA,GAAuB,OAAA,CAAQ,YAAA;EACvD,YAAA,CAAa,IAAA,UAAc,SAAA,WAAoB,OAAA;EAC/C,kBAAA,CACE,IAAA,UACA,SAAA,UACA,KAAA,UACA,aAAA,WACC,OAAA;AAAA"}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
//#region src/dirty-queue.d.ts
|
|
2
|
+
/**
|
|
3
|
+
* DirtyPathQueue — consumer-side contract the IndexFollower drains
|
|
4
|
+
* (planning/index-provider/dirty-path-follower, design §7).
|
|
5
|
+
*
|
|
6
|
+
* This is the STRUCTURAL mirror of `@aigne/afs-did-space`'s dirty-path-queue
|
|
7
|
+
* contract. The index package and the did-space package do NOT depend on each
|
|
8
|
+
* other (they meet only at the runtime wiring layer, CF/Node), so — exactly like
|
|
9
|
+
* `spaceAfs: AFSRoot` and the `IndexBackend` interface — the follower declares
|
|
10
|
+
* the shape it needs and the runtime passes a did-space `PrefixedTreeIndex`
|
|
11
|
+
* (which implements these three methods) as a structurally-compatible value.
|
|
12
|
+
* Keep this in lockstep with did-space `src/dirty-path-queue.ts`.
|
|
13
|
+
*/
|
|
14
|
+
type DirtyPathOp = "put" | "delete";
|
|
15
|
+
/**
|
|
16
|
+
* A dirty row handed to the follower. `path` is the fragment-STRIPPED
|
|
17
|
+
* `originalPath` (e.g. `items/x.json`) when the queue is a prefixed source
|
|
18
|
+
* (the normal wiring). `op/cid/latestSeq` drive scheduling + the ack race guard
|
|
19
|
+
* only — the follower re-reads live state before writing the index (design §4.3).
|
|
20
|
+
*/
|
|
21
|
+
interface DirtyPathRow {
|
|
22
|
+
path: string;
|
|
23
|
+
latestSeq: number;
|
|
24
|
+
op: DirtyPathOp;
|
|
25
|
+
cid?: string | null;
|
|
26
|
+
mtime: number;
|
|
27
|
+
attempts: number;
|
|
28
|
+
}
|
|
29
|
+
interface ListDirtyPathOptions {
|
|
30
|
+
prefix: string;
|
|
31
|
+
limit: number;
|
|
32
|
+
now?: number;
|
|
33
|
+
}
|
|
34
|
+
interface DirtyPathQueue {
|
|
35
|
+
listDirtyPaths(options: ListDirtyPathOptions): Promise<DirtyPathRow[]>;
|
|
36
|
+
ackDirtyPath(path: string, latestSeq: number): Promise<boolean>;
|
|
37
|
+
markDirtyPathError(path: string, latestSeq: number, error: string, nextAttemptAt: number): Promise<boolean>;
|
|
38
|
+
}
|
|
39
|
+
//#endregion
|
|
40
|
+
export { DirtyPathOp, DirtyPathQueue, DirtyPathRow, ListDirtyPathOptions };
|
|
41
|
+
//# sourceMappingURL=dirty-queue.d.mts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"dirty-queue.d.mts","names":[],"sources":["../src/dirty-queue.ts"],"mappings":";;AAaA;;;;;AAQA;;;;;;KARY,WAAA;;;;;;;UAQK,YAAA;EACf,IAAA;EACA,SAAA;EACA,EAAA,EAAI,WAAA;EACJ,GAAA;EACA,KAAA;EACA,QAAA;AAAA;AAAA,UAGe,oBAAA;EACf,MAAA;EACA,KAAA;EACA,GAAA;AAAA;AAAA,UAGe,cAAA;EACf,cAAA,CAAe,OAAA,EAAS,oBAAA,GAAuB,OAAA,CAAQ,YAAA;EACvD,YAAA,CAAa,IAAA,UAAc,SAAA,WAAoB,OAAA;EAC/C,kBAAA,CACE,IAAA,UACA,SAAA,UACA,KAAA,UACA,aAAA,WACC,OAAA;AAAA"}
|
package/dist/embed.cjs
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
let ufo = require("ufo");
|
|
2
|
+
|
|
3
|
+
//#region src/embed.ts
|
|
4
|
+
const DEFAULT_EMBED_TIMEOUT_MS = 2e3;
|
|
5
|
+
function embeddingTimeoutMs() {
|
|
6
|
+
const raw = (globalThis.process?.env)?.AFS_INDEX_EMBED_TIMEOUT_MS;
|
|
7
|
+
const parsed = raw ? Number(raw) : DEFAULT_EMBED_TIMEOUT_MS;
|
|
8
|
+
return Number.isFinite(parsed) && parsed > 0 ? parsed : DEFAULT_EMBED_TIMEOUT_MS;
|
|
9
|
+
}
|
|
10
|
+
async function withTimeout(promise, timeoutMs) {
|
|
11
|
+
let timer;
|
|
12
|
+
try {
|
|
13
|
+
return await Promise.race([promise, new Promise((resolve) => {
|
|
14
|
+
timer = setTimeout(() => resolve(null), timeoutMs);
|
|
15
|
+
})]);
|
|
16
|
+
} finally {
|
|
17
|
+
if (timer) clearTimeout(timer);
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* Build an embedder. Returns null (never throws) when embedding is disabled,
|
|
22
|
+
* unconfigured, or the inference call fails — callers degrade to FTS+anchor.
|
|
23
|
+
*/
|
|
24
|
+
function createEmbedder(opts) {
|
|
25
|
+
return async (text, type = "passage") => {
|
|
26
|
+
const { embeddingEnabled, llmProvider, embedRoot } = opts;
|
|
27
|
+
if (!text || !embeddingEnabled || !llmProvider || !embedRoot?.exec) return null;
|
|
28
|
+
const safeText = text.length > 6e3 ? text.slice(0, 6e3) : text;
|
|
29
|
+
try {
|
|
30
|
+
const result = await withTimeout(embedRoot.exec((0, ufo.joinURL)(llmProvider, ".actions", "embed"), {
|
|
31
|
+
text: safeText,
|
|
32
|
+
input: safeText,
|
|
33
|
+
type
|
|
34
|
+
}, {}), embeddingTimeoutMs());
|
|
35
|
+
if (!result || !result.success || !result.data) return null;
|
|
36
|
+
const data = result.data;
|
|
37
|
+
return data.embedding ?? data.embeddings?.[0] ?? null;
|
|
38
|
+
} catch {
|
|
39
|
+
return null;
|
|
40
|
+
}
|
|
41
|
+
};
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
//#endregion
|
|
45
|
+
exports.createEmbedder = createEmbedder;
|
package/dist/embed.d.cts
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import { AFSRoot } from "@aigne/afs";
|
|
2
|
+
|
|
3
|
+
//#region src/embed.d.ts
|
|
4
|
+
interface EmbedderOptions {
|
|
5
|
+
/** Master switch — when false the embedder is a no-op (returns null). */
|
|
6
|
+
embeddingEnabled?: boolean;
|
|
7
|
+
/** AFS path of the embed model node, e.g. `/dev/ai/hubs/workers-ai/models/bge-m3`. */
|
|
8
|
+
llmProvider?: string;
|
|
9
|
+
/** AFS root whose `exec` reaches `<llmProvider>/.actions/embed` (a `/dev/ai` mount). */
|
|
10
|
+
embedRoot?: AFSRoot;
|
|
11
|
+
}
|
|
12
|
+
type Embedder = (text?: string, type?: "query" | "passage") => Promise<number[] | null>;
|
|
13
|
+
/**
|
|
14
|
+
* Build an embedder. Returns null (never throws) when embedding is disabled,
|
|
15
|
+
* unconfigured, or the inference call fails — callers degrade to FTS+anchor.
|
|
16
|
+
*/
|
|
17
|
+
declare function createEmbedder(opts: EmbedderOptions): Embedder;
|
|
18
|
+
//#endregion
|
|
19
|
+
export { Embedder, EmbedderOptions, createEmbedder };
|
|
20
|
+
//# sourceMappingURL=embed.d.cts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"embed.d.cts","names":[],"sources":["../src/embed.ts"],"mappings":";;;UAYiB,eAAA;EAMI;EAJnB,gBAAA;EAOU;EALV,WAAA;;EAEA,SAAA,GAAY,OAAA;AAAA;AAAA,KAGF,QAAA,IAAY,IAAA,WAAe,IAAA,2BAA+B,OAAA;;;;AA8BtE;iBAAgB,cAAA,CAAe,IAAA,EAAM,eAAA,GAAkB,QAAA"}
|
package/dist/embed.d.mts
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import { AFSRoot } from "@aigne/afs";
|
|
2
|
+
|
|
3
|
+
//#region src/embed.d.ts
|
|
4
|
+
interface EmbedderOptions {
|
|
5
|
+
/** Master switch — when false the embedder is a no-op (returns null). */
|
|
6
|
+
embeddingEnabled?: boolean;
|
|
7
|
+
/** AFS path of the embed model node, e.g. `/dev/ai/hubs/workers-ai/models/bge-m3`. */
|
|
8
|
+
llmProvider?: string;
|
|
9
|
+
/** AFS root whose `exec` reaches `<llmProvider>/.actions/embed` (a `/dev/ai` mount). */
|
|
10
|
+
embedRoot?: AFSRoot;
|
|
11
|
+
}
|
|
12
|
+
type Embedder = (text?: string, type?: "query" | "passage") => Promise<number[] | null>;
|
|
13
|
+
/**
|
|
14
|
+
* Build an embedder. Returns null (never throws) when embedding is disabled,
|
|
15
|
+
* unconfigured, or the inference call fails — callers degrade to FTS+anchor.
|
|
16
|
+
*/
|
|
17
|
+
declare function createEmbedder(opts: EmbedderOptions): Embedder;
|
|
18
|
+
//#endregion
|
|
19
|
+
export { Embedder, EmbedderOptions, createEmbedder };
|
|
20
|
+
//# sourceMappingURL=embed.d.mts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"embed.d.mts","names":[],"sources":["../src/embed.ts"],"mappings":";;;UAYiB,eAAA;EAMI;EAJnB,gBAAA;EAOU;EALV,WAAA;;EAEA,SAAA,GAAY,OAAA;AAAA;AAAA,KAGF,QAAA,IAAY,IAAA,WAAe,IAAA,2BAA+B,OAAA;;;;AA8BtE;iBAAgB,cAAA,CAAe,IAAA,EAAM,eAAA,GAAkB,QAAA"}
|
package/dist/embed.mjs
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import { joinURL } from "ufo";
|
|
2
|
+
|
|
3
|
+
//#region src/embed.ts
|
|
4
|
+
const DEFAULT_EMBED_TIMEOUT_MS = 2e3;
|
|
5
|
+
function embeddingTimeoutMs() {
|
|
6
|
+
const raw = (globalThis.process?.env)?.AFS_INDEX_EMBED_TIMEOUT_MS;
|
|
7
|
+
const parsed = raw ? Number(raw) : DEFAULT_EMBED_TIMEOUT_MS;
|
|
8
|
+
return Number.isFinite(parsed) && parsed > 0 ? parsed : DEFAULT_EMBED_TIMEOUT_MS;
|
|
9
|
+
}
|
|
10
|
+
async function withTimeout(promise, timeoutMs) {
|
|
11
|
+
let timer;
|
|
12
|
+
try {
|
|
13
|
+
return await Promise.race([promise, new Promise((resolve) => {
|
|
14
|
+
timer = setTimeout(() => resolve(null), timeoutMs);
|
|
15
|
+
})]);
|
|
16
|
+
} finally {
|
|
17
|
+
if (timer) clearTimeout(timer);
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* Build an embedder. Returns null (never throws) when embedding is disabled,
|
|
22
|
+
* unconfigured, or the inference call fails — callers degrade to FTS+anchor.
|
|
23
|
+
*/
|
|
24
|
+
function createEmbedder(opts) {
|
|
25
|
+
return async (text, type = "passage") => {
|
|
26
|
+
const { embeddingEnabled, llmProvider, embedRoot } = opts;
|
|
27
|
+
if (!text || !embeddingEnabled || !llmProvider || !embedRoot?.exec) return null;
|
|
28
|
+
const safeText = text.length > 6e3 ? text.slice(0, 6e3) : text;
|
|
29
|
+
try {
|
|
30
|
+
const result = await withTimeout(embedRoot.exec(joinURL(llmProvider, ".actions", "embed"), {
|
|
31
|
+
text: safeText,
|
|
32
|
+
input: safeText,
|
|
33
|
+
type
|
|
34
|
+
}, {}), embeddingTimeoutMs());
|
|
35
|
+
if (!result || !result.success || !result.data) return null;
|
|
36
|
+
const data = result.data;
|
|
37
|
+
return data.embedding ?? data.embeddings?.[0] ?? null;
|
|
38
|
+
} catch {
|
|
39
|
+
return null;
|
|
40
|
+
}
|
|
41
|
+
};
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
//#endregion
|
|
45
|
+
export { createEmbedder };
|
|
46
|
+
//# sourceMappingURL=embed.mjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"embed.mjs","names":[],"sources":["../src/embed.ts"],"sourcesContent":["/**\n * Shared passage/query embedder for the semantic index (Epic #932).\n *\n * Both the query path (`AFSIndex.autoEmbed`) and the two index paths — the\n * `index` action AND the changelog follower — need to turn text into a vector\n * via the SAME `<llmProvider>/.actions/embed` call. Extracted here so the follower\n * (and the CF cron's `rebuildFollowerFromCursor`, which has no `AFSIndex` handle)\n * can embed identically without duplicating the request/response shape handling.\n */\nimport type { AFSRoot } from \"@aigne/afs\";\nimport { joinURL } from \"ufo\";\n\nexport interface EmbedderOptions {\n /** Master switch — when false the embedder is a no-op (returns null). */\n embeddingEnabled?: boolean;\n /** AFS path of the embed model node, e.g. `/dev/ai/hubs/workers-ai/models/bge-m3`. */\n llmProvider?: string;\n /** AFS root whose `exec` reaches `<llmProvider>/.actions/embed` (a `/dev/ai` mount). */\n embedRoot?: AFSRoot;\n}\n\nexport type Embedder = (text?: string, type?: \"query\" | \"passage\") => Promise<number[] | null>;\n\nconst DEFAULT_EMBED_TIMEOUT_MS = 2_000;\n\nfunction embeddingTimeoutMs(): number {\n const env = (globalThis as { process?: { env?: Record<string, string | undefined> } }).process\n ?.env;\n const raw = env?.AFS_INDEX_EMBED_TIMEOUT_MS;\n const parsed = raw ? Number(raw) : DEFAULT_EMBED_TIMEOUT_MS;\n return Number.isFinite(parsed) && parsed > 0 ? parsed : DEFAULT_EMBED_TIMEOUT_MS;\n}\n\nasync function withTimeout<T>(promise: Promise<T>, timeoutMs: number): Promise<T | null> {\n let timer: ReturnType<typeof setTimeout> | undefined;\n try {\n return await Promise.race([\n promise,\n new Promise<null>((resolve) => {\n timer = setTimeout(() => resolve(null), timeoutMs);\n }),\n ]);\n } finally {\n if (timer) clearTimeout(timer);\n }\n}\n\n/**\n * Build an embedder. Returns null (never throws) when embedding is disabled,\n * unconfigured, or the inference call fails — callers degrade to FTS+anchor.\n */\nexport function createEmbedder(opts: EmbedderOptions): Embedder {\n return async (text, type = \"passage\") => {\n const { embeddingEnabled, llmProvider, embedRoot } = opts;\n if (!text || !embeddingEnabled || !llmProvider || !embedRoot?.exec) {\n return null;\n }\n // text-embedding-3-* / bge-m3 have ~8K-token contexts; ~6 chars/token, so\n // 6000 chars stays well inside the cap so a verbose body doesn't 400.\n const safeText = text.length > 6000 ? text.slice(0, 6000) : text;\n try {\n // Superset args: legacy/E5 `{ text, type }` → `{ embedding }`, and\n // AigneHub/OpenAI `{ input }` → `{ embeddings: [[...]] }`. Accept either.\n const result = await withTimeout(\n embedRoot.exec(\n joinURL(llmProvider, \".actions\", \"embed\"),\n {\n text: safeText,\n input: safeText,\n type,\n },\n {},\n ),\n embeddingTimeoutMs(),\n );\n if (!result || !result.success || !result.data) return null;\n const data = result.data as { embedding?: number[]; embeddings?: number[][] };\n return data.embedding ?? data.embeddings?.[0] ?? null;\n } catch {\n return null;\n }\n };\n}\n"],"mappings":";;;AAuBA,MAAM,2BAA2B;AAEjC,SAAS,qBAA6B;CAGpC,MAAM,OAFO,WAA0E,SACnF,MACa;CACjB,MAAM,SAAS,MAAM,OAAO,IAAI,GAAG;AACnC,QAAO,OAAO,SAAS,OAAO,IAAI,SAAS,IAAI,SAAS;;AAG1D,eAAe,YAAe,SAAqB,WAAsC;CACvF,IAAI;AACJ,KAAI;AACF,SAAO,MAAM,QAAQ,KAAK,CACxB,SACA,IAAI,SAAe,YAAY;AAC7B,WAAQ,iBAAiB,QAAQ,KAAK,EAAE,UAAU;IAClD,CACH,CAAC;WACM;AACR,MAAI,MAAO,cAAa,MAAM;;;;;;;AAQlC,SAAgB,eAAe,MAAiC;AAC9D,QAAO,OAAO,MAAM,OAAO,cAAc;EACvC,MAAM,EAAE,kBAAkB,aAAa,cAAc;AACrD,MAAI,CAAC,QAAQ,CAAC,oBAAoB,CAAC,eAAe,CAAC,WAAW,KAC5D,QAAO;EAIT,MAAM,WAAW,KAAK,SAAS,MAAO,KAAK,MAAM,GAAG,IAAK,GAAG;AAC5D,MAAI;GAGF,MAAM,SAAS,MAAM,YACnB,UAAU,KACR,QAAQ,aAAa,YAAY,QAAQ,EACzC;IACE,MAAM;IACN,OAAO;IACP;IACD,EACD,EAAE,CACH,EACD,oBAAoB,CACrB;AACD,OAAI,CAAC,UAAU,CAAC,OAAO,WAAW,CAAC,OAAO,KAAM,QAAO;GACvD,MAAM,OAAO,OAAO;AACpB,UAAO,KAAK,aAAa,KAAK,aAAa,MAAM;UAC3C;AACN,UAAO"}
|