@flaxia/node 0.3.7 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +43 -19
- package/dist/__tests__/consent-policy.test.d.ts +2 -0
- package/dist/__tests__/consent-policy.test.d.ts.map +1 -0
- package/dist/assets/VectorStoreEngine.js +140 -85
- package/dist/assets/ai-inference.js +173 -33
- package/dist/assets/container.js +368 -260
- package/dist/assets/dist.js +209 -0
- package/dist/assets/egress-guard.js +174 -0
- package/dist/assets/image-process.js +184 -19
- package/dist/assets/nudenet.js +2 -2
- package/dist/client/SignalingClient.d.ts +6 -0
- package/dist/client/SignalingClient.d.ts.map +1 -1
- package/dist/consent/ConsentUI.d.ts +44 -5
- package/dist/consent/ConsentUI.d.ts.map +1 -1
- package/dist/consent/notice.d.ts +29 -0
- package/dist/consent/notice.d.ts.map +1 -0
- package/dist/consent/storage.d.ts +91 -7
- package/dist/consent/storage.d.ts.map +1 -1
- package/dist/executor/WorkerPool.d.ts +35 -1
- package/dist/executor/WorkerPool.d.ts.map +1 -1
- package/dist/executor/__tests__/WorkerPool.swarm.test.d.ts +13 -0
- package/dist/executor/__tests__/WorkerPool.swarm.test.d.ts.map +1 -0
- package/dist/executor/__tests__/container-executor.test.d.ts +2 -0
- package/dist/executor/__tests__/container-executor.test.d.ts.map +1 -0
- package/dist/executor/__tests__/egress-guard.test.d.ts +2 -0
- package/dist/executor/__tests__/egress-guard.test.d.ts.map +1 -0
- package/dist/executor/__tests__/webgpuProbe.test.d.ts +2 -0
- package/dist/executor/__tests__/webgpuProbe.test.d.ts.map +1 -0
- package/dist/executor/container-executor.d.ts +49 -1
- package/dist/executor/container-executor.d.ts.map +1 -1
- package/dist/executor/egress-guard.d.ts +105 -0
- package/dist/executor/egress-guard.d.ts.map +1 -0
- package/dist/executor/memoryProbe.d.ts +25 -0
- package/dist/executor/memoryProbe.d.ts.map +1 -1
- package/dist/executor/throttle.d.ts +62 -4
- package/dist/executor/throttle.d.ts.map +1 -1
- package/dist/executor/webgpuProbe.d.ts +9 -0
- package/dist/executor/webgpuProbe.d.ts.map +1 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +766 -230
- package/dist/swarm/__tests__/adapter.test.d.ts +2 -0
- package/dist/swarm/__tests__/adapter.test.d.ts.map +1 -0
- package/dist/swarm/__tests__/controller.test.d.ts +2 -0
- package/dist/swarm/__tests__/controller.test.d.ts.map +1 -0
- package/dist/swarm/__tests__/defects.test.d.ts +16 -0
- package/dist/swarm/__tests__/defects.test.d.ts.map +1 -0
- package/dist/swarm/__tests__/hardening.test.d.ts +2 -0
- package/dist/swarm/__tests__/hardening.test.d.ts.map +1 -0
- package/dist/swarm/__tests__/runtime.test.d.ts +2 -0
- package/dist/swarm/__tests__/runtime.test.d.ts.map +1 -0
- package/dist/swarm/__tests__/session.test.d.ts +2 -0
- package/dist/swarm/__tests__/session.test.d.ts.map +1 -0
- package/dist/swarm/adapter.d.ts +115 -0
- package/dist/swarm/adapter.d.ts.map +1 -0
- package/dist/swarm/controller.d.ts +87 -0
- package/dist/swarm/controller.d.ts.map +1 -0
- package/dist/swarm/messages.d.ts +41 -0
- package/dist/swarm/messages.d.ts.map +1 -0
- package/dist/swarm/runtime.d.ts +16 -0
- package/dist/swarm/runtime.d.ts.map +1 -0
- package/dist/swarm/session.d.ts +73 -0
- package/dist/swarm/session.d.ts.map +1 -0
- package/dist/swarm-engine.js +7477 -0
- package/dist/vector-store/HNSWIndex.d.ts +30 -0
- package/dist/vector-store/HNSWIndex.d.ts.map +1 -1
- package/dist/vector-store/VectorStoreEngine.d.ts +43 -0
- package/dist/vector-store/VectorStoreEngine.d.ts.map +1 -1
- package/dist/vector-store/__tests__/HNSWIndex.test.d.ts +2 -0
- package/dist/vector-store/__tests__/HNSWIndex.test.d.ts.map +1 -0
- package/dist/vector-store/__tests__/VectorStoreEngine.test.d.ts +2 -0
- package/dist/vector-store/__tests__/VectorStoreEngine.test.d.ts.map +1 -0
- package/dist/worker/__tests__/main.worker.abort.test.d.ts +2 -0
- package/dist/worker/__tests__/main.worker.abort.test.d.ts.map +1 -0
- package/dist/worker/__tests__/main.worker.control.test.d.ts +2 -0
- package/dist/worker/__tests__/main.worker.control.test.d.ts.map +1 -0
- package/dist/worker.js +643 -67
- package/dist/workloads/__tests__/nudenet.test.d.ts +2 -0
- package/dist/workloads/__tests__/nudenet.test.d.ts.map +1 -0
- package/dist/workloads/ai-inference.d.ts +2 -1
- package/dist/workloads/ai-inference.d.ts.map +1 -1
- package/dist/workloads/ai-model-registry.d.ts +72 -0
- package/dist/workloads/ai-model-registry.d.ts.map +1 -0
- package/dist/workloads/container.d.ts +1 -1
- package/dist/workloads/container.d.ts.map +1 -1
- package/dist/workloads/image-process.d.ts +27 -0
- package/dist/workloads/image-process.d.ts.map +1 -1
- package/dist/workloads/nudenet.d.ts +8 -0
- package/dist/workloads/nudenet.d.ts.map +1 -1
- package/package.json +3 -2
- package/dist/assets/memoryProbe.js +0 -28
- package/dist/client/WebRTCPeer.d.ts +0 -8
- package/dist/client/WebRTCPeer.d.ts.map +0 -1
package/README.md
CHANGED
|
@@ -36,17 +36,27 @@ const controller = initFlaxiaNode({
|
|
|
36
36
|
| `consent` | `ConsentConfig` | yes | 同意UIの設定 |
|
|
37
37
|
| `consent.brandName` | `string` | yes | サイト名(内蔵UIに表示) |
|
|
38
38
|
| `consent.position` | `'bottom-right' \| 'bottom-left' \| 'top-right' \| 'top-left'` | yes | 内蔵UIの表示位置 |
|
|
39
|
-
| `consent.accentColor` | `string` | no | 内蔵UI
|
|
39
|
+
| `consent.accentColor` | `string` | no | 内蔵UIのアクセントカラー(許可リスト外は既定色) |
|
|
40
|
+
| `consent.acceptLabel` / `consent.rejectLabel` | `string` | no | ボタン文言の上書き(テキストとして描画) |
|
|
41
|
+
| `consent.privacyPolicyUrl` | `string` | no | プライバシーポリシーへのリンク(http/https のみ) |
|
|
42
|
+
| `consent.minVisibleMs` | `number` | no | 同意ボタンが有効になるまでの最低表示時間。**延長のみ**(下限1500ms) |
|
|
40
43
|
| `consent.onConsentRequired` | `(controls: ConsentControls) => void` | no | ホスト独自の同意UIを使う場合に指定。指定すると内蔵バナーは表示されない |
|
|
41
44
|
| `maxCpuLoad` | `number` | no | CPU負荷制限(未実装) |
|
|
42
45
|
|
|
43
|
-
### ホスト独自の同意UI
|
|
46
|
+
### ホスト独自の同意UI(オプトインが必要)
|
|
44
47
|
|
|
45
48
|
`consent.onConsentRequired` を渡すと、内蔵の Shadow DOM バナーは使われず、
|
|
46
|
-
同意状態が `unset`
|
|
47
|
-
|
|
49
|
+
同意状態が `unset` のときだけホストが呼び出されます。
|
|
50
|
+
|
|
51
|
+
ただし既定では **`accept()` を呼んでも同意は保存されません**。訪問者に実際の
|
|
52
|
+
開示文を見せて操作させるのはホストの責任になるため、ホスト独自UIを使う場合は
|
|
53
|
+
先に `setFlaxiaNodeHostManagedConsent(true)` で明示的にオプトインしてください。
|
|
48
54
|
|
|
49
55
|
```typescript
|
|
56
|
+
import { initFlaxiaNode, setFlaxiaNodeHostManagedConsent } from '@flaxia/node'
|
|
57
|
+
|
|
58
|
+
setFlaxiaNodeHostManagedConsent(true) // ホストが開示文を提示する責任を負う
|
|
59
|
+
|
|
50
60
|
initFlaxiaNode({
|
|
51
61
|
orchestratorUrl: 'https://crowd.flaxia.app',
|
|
52
62
|
siteId: 'your-site-id',
|
|
@@ -54,7 +64,8 @@ initFlaxiaNode({
|
|
|
54
64
|
brandName: 'あなたのサービス名',
|
|
55
65
|
position: 'bottom-right',
|
|
56
66
|
onConsentRequired: ({ state, accept, reject }) => {
|
|
57
|
-
//
|
|
67
|
+
// 現在の開示文(packages/node/docs/01-consent-ui.md)を提示し、
|
|
68
|
+
// ユーザーの選択に応じて accept() / reject()
|
|
58
69
|
showMyConsentModal({
|
|
59
70
|
onAccept: accept,
|
|
60
71
|
onReject: reject,
|
|
@@ -64,6 +75,9 @@ initFlaxiaNode({
|
|
|
64
75
|
})
|
|
65
76
|
```
|
|
66
77
|
|
|
78
|
+
オプトインせずに `accept()` を呼んだ場合は `console.warn` が出て、状態は
|
|
79
|
+
`unset` のまま(=同意なし)になります。
|
|
80
|
+
|
|
67
81
|
### コントローラ(設定画面からの変更)
|
|
68
82
|
|
|
69
83
|
`initFlaxiaNode()` は `FlaxiaNodeController` を返します。同一モジュール内で一度だけ
|
|
@@ -73,27 +87,36 @@ initFlaxiaNode({
|
|
|
73
87
|
const controller = initFlaxiaNode({ /* ... */ })
|
|
74
88
|
|
|
75
89
|
controller.getConsentState() // 'unset' | 'granted' | 'denied'
|
|
76
|
-
controller.grant() //
|
|
90
|
+
controller.grant() // 同意を保存(拒否を消去)※ホスト管理UIのオプトインが必要
|
|
77
91
|
controller.deny() // 拒否を保存して停止(同意を消去)
|
|
78
92
|
controller.start() // ノード起動
|
|
79
93
|
controller.stop() // ノード停止 + Web Worker 解放
|
|
80
94
|
controller.isRunning() // 稼働中か
|
|
81
|
-
controller.clearConsent() //
|
|
95
|
+
controller.clearConsent() // 同意・拒否・HMAC鍵をすべて消去
|
|
82
96
|
```
|
|
83
97
|
|
|
84
|
-
同意は localStorage
|
|
85
|
-
|
|
98
|
+
同意は `localStorage` の `flaxia_consent_record` に保存され、**180日**で失効します
|
|
99
|
+
(`grant` と `deny` は排他)。レコードは HMAC-SHA-256 で署名され、ページの
|
|
100
|
+
オリジンと開示文バージョンに束縛されるため、`localStorage` に文字列を書いただけの
|
|
101
|
+
偽造や、旧開示文に対する同意は「同意なし」として扱われます。
|
|
86
102
|
|
|
87
103
|
### 保存状態の読み取り(ノード初期化前)
|
|
88
104
|
|
|
89
105
|
`getFlaxiaNodeConsentState()` はコントローラを作らず、ネットワークにも触れず、
|
|
90
|
-
|
|
91
|
-
|
|
106
|
+
保存済みの同意状態を同期的に返します。**fail closed** で、HMAC 検証が完了して
|
|
107
|
+
いなければ `'granted'` は返しません。
|
|
108
|
+
|
|
109
|
+
リロード後の自動再開を維持したいホストは、**`initFlaxiaNode()` を呼ぶ前に**
|
|
110
|
+
`await initFlaxiaNodeConsent()` を実行してください。await せずに初期化すると、
|
|
111
|
+
再訪した同意済みの訪問者にもバナーが表示され、ノードは自動起動しません
|
|
112
|
+
(同意が無いものとして扱われるため)。
|
|
92
113
|
|
|
93
114
|
```typescript
|
|
94
|
-
import { getFlaxiaNodeConsentState } from '@flaxia/node'
|
|
115
|
+
import { getFlaxiaNodeConsentState, initFlaxiaNode, initFlaxiaNodeConsent } from '@flaxia/node'
|
|
95
116
|
|
|
96
|
-
|
|
117
|
+
await initFlaxiaNodeConsent() // 冪等・例外を投げない
|
|
118
|
+
if (getFlaxiaNodeConsentState() !== 'granted') showSettingsToggle()
|
|
119
|
+
initFlaxiaNode({ /* ... */ })
|
|
97
120
|
```
|
|
98
121
|
|
|
99
122
|
## アーキテクチャ
|
|
@@ -105,7 +128,7 @@ getFlaxiaNodeConsentState() // 'unset' | 'granted' | 'denied'
|
|
|
105
128
|
│ ┌──────────┐ ┌──────────────┐ │
|
|
106
129
|
│ │ ConsentUI│ │ Signaling │ │
|
|
107
130
|
│ │ (Shadow │ │ Client │──WebSocket─┼──→ Worker
|
|
108
|
-
│ │ DOM) │ │ (
|
|
131
|
+
│ │ DOM) │ │ (WebSocket) │ │
|
|
109
132
|
│ └──────────┘ └──────┬───────┘ │
|
|
110
133
|
│ │ │
|
|
111
134
|
│ ┌─────▼──────┐ │
|
|
@@ -133,11 +156,11 @@ getFlaxiaNodeConsentState() // 'unset' | 'granted' | 'denied'
|
|
|
133
156
|
|
|
134
157
|
| レイヤー | ディレクトリ | 責務 |
|
|
135
158
|
|---------|------------|------|
|
|
136
|
-
| **同意** | `src/consent/` | Shadow DOM
|
|
137
|
-
| **接続** | `src/client/` | WebSocket signaling
|
|
159
|
+
| **同意** | `src/consent/` | closed Shadow DOM の同意バナー、HMAC署名付き同意レコードの保存・検証 |
|
|
160
|
+
| **接続** | `src/client/` | WebSocket signaling(登録・再接続・タスク授受) |
|
|
138
161
|
| **実行** | `src/executor/` | WebWorker のライフサイクル管理、タイムアウト処理、WASM コンテナ実行 |
|
|
139
162
|
| **Worker** | `src/worker/` | WebWorker エントリ、workload への動的ディスパッチ |
|
|
140
|
-
| **Workload** | `src/workloads/` | AI
|
|
163
|
+
| **Workload** | `src/workloads/` | AI推論・画像処理・NSFW判定・コンテナ実行の実装 |
|
|
141
164
|
|
|
142
165
|
## 対応ワークロード
|
|
143
166
|
|
|
@@ -146,11 +169,12 @@ getFlaxiaNodeConsentState() // 'unset' | 'granted' | 'denied'
|
|
|
146
169
|
| `ai-inference` | `workloads/ai-inference.ts` | 🤗 Transformers.js (ONNX Runtime) |
|
|
147
170
|
| `image-process` | `workloads/image-process.ts` | OffscreenCanvas API |
|
|
148
171
|
| `container` | `workloads/container.ts` | container2wasm + WASI |
|
|
172
|
+
| `nudenet` | `workloads/nudenet.ts` | NSFW(成人向け)画像判定 |
|
|
149
173
|
|
|
150
174
|
## フロー
|
|
151
175
|
|
|
152
|
-
1. `initFlaxiaNode()` が呼ばれると、
|
|
153
|
-
2. 未同意の場合、内蔵の Shadow DOM 同意バナー、または `onConsentRequired` でホストのUIを表示
|
|
176
|
+
1. `initFlaxiaNode()` が呼ばれると、HMAC署名付きの同意レコードを確認
|
|
177
|
+
2. 未同意の場合、内蔵の closed Shadow DOM 同意バナー、または `onConsentRequired` でホストのUIを表示
|
|
154
178
|
3. 同意後、WebSocket でオーケストレーターに接続(`/crowd/signal`)
|
|
155
179
|
4. タスクを受信すると WebWorker で実行
|
|
156
180
|
5. 結果・中間トークンを WebSocket で返送
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"consent-policy.test.d.ts","sourceRoot":"","sources":["../../src/__tests__/consent-policy.test.ts"],"names":[],"mappings":""}
|
|
@@ -1,47 +1,52 @@
|
|
|
1
1
|
//#region src/vector-store/HNSWIndex.ts
|
|
2
|
-
var e = class {
|
|
2
|
+
var e = 1e5, t = 1024, n = class {
|
|
3
3
|
constructor(e, t = "cosine", n = 16, r = 200) {
|
|
4
4
|
this.dimensions = e, this.metric = t, this.M = n, this.efConstruction = r, this.nodes = /* @__PURE__ */ new Map(), this.docIdMap = /* @__PURE__ */ new Map(), this.enterPoint = null, this.nextNodeId = 0, this.maxLevel = 0;
|
|
5
5
|
}
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
6
|
+
get dimensionsValue() {
|
|
7
|
+
return this.dimensions;
|
|
8
|
+
}
|
|
9
|
+
insert(t, n) {
|
|
10
|
+
if (this.nodes.size >= 1e5) throw Error(`HNSW index is full (${e} nodes); refusing to insert more`);
|
|
11
|
+
if (!(n instanceof Float32Array) || n.length !== this.dimensions) throw Error(`HNSW insert: vector must be a Float32Array of ${this.dimensions} dimensions, got ${n?.length ?? "none"}`);
|
|
12
|
+
let r = this.nextNodeId++, i = this.randomLevel(), a = {
|
|
13
|
+
id: r,
|
|
14
|
+
docId: t,
|
|
15
|
+
vector: n,
|
|
16
|
+
level: i,
|
|
12
17
|
neighbors: /* @__PURE__ */ new Map()
|
|
13
18
|
};
|
|
14
|
-
if (this.nodes.set(
|
|
15
|
-
this.enterPoint =
|
|
19
|
+
if (this.nodes.set(r, a), this.docIdMap.set(t, r), this.enterPoint === null) {
|
|
20
|
+
this.enterPoint = r, this.maxLevel = i;
|
|
16
21
|
return;
|
|
17
22
|
}
|
|
18
|
-
let
|
|
19
|
-
for (let e = this.maxLevel; e >
|
|
20
|
-
let
|
|
21
|
-
for (;
|
|
22
|
-
|
|
23
|
-
let r =
|
|
23
|
+
let o = this.nodes.get(this.enterPoint), s = this.distance(n, o.vector);
|
|
24
|
+
for (let e = this.maxLevel; e > i; e--) {
|
|
25
|
+
let t = !0;
|
|
26
|
+
for (; t;) {
|
|
27
|
+
t = !1;
|
|
28
|
+
let r = o.neighbors.get(e) || [];
|
|
24
29
|
for (let e of r) {
|
|
25
30
|
let r = this.nodes.get(e);
|
|
26
31
|
if (!r) continue;
|
|
27
|
-
let i = this.distance(
|
|
28
|
-
i <
|
|
32
|
+
let i = this.distance(n, r.vector);
|
|
33
|
+
i < s && (s = i, o = r, t = !0);
|
|
29
34
|
}
|
|
30
35
|
}
|
|
31
36
|
}
|
|
32
|
-
for (let e = Math.min(
|
|
33
|
-
let
|
|
34
|
-
for (let e of i)
|
|
35
|
-
|
|
37
|
+
for (let e = Math.min(i, this.maxLevel); e >= 0; e--) {
|
|
38
|
+
let t = this.searchLayer(n, o, e, this.efConstruction), i = this.selectNeighbors(t, this.M), a = o.neighbors.get(e) || [];
|
|
39
|
+
for (let e of i) a.includes(e.nodeId) || a.push(e.nodeId);
|
|
40
|
+
o.neighbors.set(e, a);
|
|
36
41
|
for (let t of i) {
|
|
37
|
-
let
|
|
38
|
-
if (
|
|
39
|
-
let t =
|
|
40
|
-
t.includes(
|
|
42
|
+
let n = this.nodes.get(t.nodeId);
|
|
43
|
+
if (n) {
|
|
44
|
+
let t = n.neighbors.get(e) || [];
|
|
45
|
+
t.includes(r) || t.push(r), n.neighbors.set(e, t.slice(-this.M));
|
|
41
46
|
}
|
|
42
47
|
}
|
|
43
48
|
}
|
|
44
|
-
|
|
49
|
+
i > this.maxLevel && (this.enterPoint = r, this.maxLevel = i);
|
|
45
50
|
}
|
|
46
51
|
search(e, t) {
|
|
47
52
|
if (this.nodes.size === 0 || this.enterPoint === null) return [];
|
|
@@ -102,6 +107,7 @@ var e = class {
|
|
|
102
107
|
return e.sort((e, t) => e.distance - t.distance).slice(0, t);
|
|
103
108
|
}
|
|
104
109
|
distance(e, t) {
|
|
110
|
+
if (e.length !== t.length) throw Error(`HNSW distance: dimension mismatch (${e.length} vs ${t.length})`);
|
|
105
111
|
if (this.metric === "cosine") {
|
|
106
112
|
let n = 0, r = 0, i = 0;
|
|
107
113
|
for (let a = 0; a < e.length; a++) n += e[a] * t[a], r += e[a] * e[a], i += t[a] * t[a];
|
|
@@ -134,70 +140,112 @@ var e = class {
|
|
|
134
140
|
}
|
|
135
141
|
return e;
|
|
136
142
|
}
|
|
137
|
-
importNodes(
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
143
|
+
importNodes(n, r) {
|
|
144
|
+
if (!Array.isArray(n)) throw Error("HNSW import: nodes must be an array");
|
|
145
|
+
if (n.length > 1e5) throw Error(`HNSW import: ${n.length} nodes exceed the ${e} quota`);
|
|
146
|
+
let i = /* @__PURE__ */ new Set();
|
|
147
|
+
for (let e of n) {
|
|
148
|
+
if (!e || typeof e != "object") throw Error("HNSW import: node entry must be an object");
|
|
149
|
+
if (!Number.isInteger(e.id) || e.id < 0) throw Error(`HNSW import: node id must be a non-negative integer, got ${JSON.stringify(e.id)}`);
|
|
150
|
+
if (i.has(e.id)) throw Error(`HNSW import: duplicate node id ${e.id}`);
|
|
151
|
+
if (i.add(e.id), typeof e.docId != "string" || !e.docId) throw Error(`HNSW import: node ${e.id} has an invalid docId`);
|
|
152
|
+
if (!Number.isInteger(e.level) || e.level < 0 || e.level > 16) throw Error(`HNSW import: node ${e.id} level ${JSON.stringify(e.level)} is outside 0..16`);
|
|
153
|
+
let t = r.get(e.id);
|
|
154
|
+
if (!t) throw Error(`HNSW import: node ${e.id} has no vector`);
|
|
155
|
+
if (!(t instanceof Float32Array) || t.length !== this.dimensions) throw Error(`HNSW import: node ${e.id} vector must be a Float32Array of ${this.dimensions} dimensions`);
|
|
156
|
+
for (let n = 0; n < t.length; n++) if (!Number.isFinite(t[n])) throw Error(`HNSW import: node ${e.id} vector contains a non-finite value at ${n}`);
|
|
157
|
+
}
|
|
158
|
+
for (let e of n) {
|
|
159
|
+
let n = e.neighbors;
|
|
160
|
+
if (!n || typeof n != "object") throw Error(`HNSW import: node ${e.id} has invalid neighbors`);
|
|
161
|
+
for (let [r, a] of Object.entries(n)) {
|
|
162
|
+
let n = Number(r);
|
|
163
|
+
if (!Number.isInteger(n) || n < 0 || n > e.level) throw Error(`HNSW import: node ${e.id} has adjacency for level ${r} above its level ${e.level}`);
|
|
164
|
+
if (!Array.isArray(a)) throw Error(`HNSW import: node ${e.id} level ${r} adjacency must be an array`);
|
|
165
|
+
if (a.length > 1024) throw Error(`HNSW import: node ${e.id} level ${r} has ${a.length} neighbors, above the ${t} limit`);
|
|
166
|
+
for (let t of a) if (!Number.isInteger(t) || !i.has(t)) throw Error(`HNSW import: node ${e.id} references unknown neighbor ${JSON.stringify(t)}`);
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
for (let e of n) {
|
|
170
|
+
let t = /* @__PURE__ */ new Map();
|
|
171
|
+
for (let [n, r] of Object.entries(e.neighbors)) t.set(Number(n), [...r]);
|
|
172
|
+
let n = {
|
|
173
|
+
id: e.id,
|
|
174
|
+
docId: e.docId,
|
|
175
|
+
vector: r.get(e.id),
|
|
176
|
+
level: e.level,
|
|
177
|
+
neighbors: t
|
|
147
178
|
};
|
|
148
|
-
this.nodes.set(
|
|
179
|
+
this.nodes.set(e.id, n), this.docIdMap.set(e.docId, e.id), e.id >= this.nextNodeId && (this.nextNodeId = e.id + 1), e.level > this.maxLevel && (this.maxLevel = e.level);
|
|
149
180
|
}
|
|
150
181
|
this.nodes.size > 0 && this.enterPoint === null && (this.enterPoint = this.nodes.keys().next().value);
|
|
151
182
|
}
|
|
152
|
-
};
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
183
|
+
}, r = 4096, i = 1048576, a = e, o = 1e3;
|
|
184
|
+
function s(e, t, n = r, a = i) {
|
|
185
|
+
if (!Array.isArray(e)) throw Error(`vector-store: ${t} must be an array of numbers`);
|
|
186
|
+
let o = e.length;
|
|
187
|
+
if (!Number.isInteger(o) || o <= 0) throw Error(`vector-store: ${t} length must be a positive integer, got ${String(o)}`);
|
|
188
|
+
if (o > n) throw Error(`vector-store: ${t} has ${o} dimensions, above the ${n} limit`);
|
|
189
|
+
if (o > a) throw Error(`vector-store: ${t} has ${o} elements, above the ${a} limit`);
|
|
190
|
+
for (let n = 0; n < o; n++) {
|
|
191
|
+
let r = e[n];
|
|
192
|
+
if (typeof r != "number" || !Number.isFinite(r)) throw Error(`vector-store: ${t}[${n}] must be a finite number`);
|
|
193
|
+
}
|
|
194
|
+
return o;
|
|
195
|
+
}
|
|
196
|
+
function c(e, t, n) {
|
|
197
|
+
if (typeof e != "string" && typeof e != "number") throw Error(`vector-store: shardKey must be a string or number, got ${typeof e}`);
|
|
198
|
+
let r = typeof e == "number" ? e : Number(e);
|
|
199
|
+
if (!Number.isInteger(r)) throw Error(`vector-store: shardKey must be an integer, got ${JSON.stringify(e)}`);
|
|
200
|
+
if (r < t || r > n) throw Error(`Shard key ${r} out of range`);
|
|
201
|
+
return r;
|
|
202
|
+
}
|
|
203
|
+
function l(e, t, n) {
|
|
156
204
|
return new Promise((r, i) => {
|
|
157
205
|
let a = indexedDB.open(e, t);
|
|
158
206
|
a.onupgradeneeded = () => n(a.result), a.onsuccess = () => r(a.result), a.onerror = () => i(a.error);
|
|
159
207
|
});
|
|
160
208
|
}
|
|
161
|
-
function
|
|
209
|
+
function u(e, t, n = "readonly") {
|
|
162
210
|
return e.transaction(t, n).objectStore(t);
|
|
163
211
|
}
|
|
164
|
-
function
|
|
165
|
-
return new Promise((
|
|
166
|
-
let
|
|
167
|
-
|
|
212
|
+
function d(e, t, n) {
|
|
213
|
+
return new Promise((r, i) => {
|
|
214
|
+
let a = u(e, t).get(n);
|
|
215
|
+
a.onsuccess = () => r(a.result), a.onerror = () => i(a.error);
|
|
168
216
|
});
|
|
169
217
|
}
|
|
170
|
-
function
|
|
171
|
-
return new Promise((
|
|
172
|
-
let
|
|
173
|
-
|
|
218
|
+
function f(e, t, n) {
|
|
219
|
+
return new Promise((r, i) => {
|
|
220
|
+
let a = u(e, t, "readwrite").put(n);
|
|
221
|
+
a.onsuccess = () => r(), a.onerror = () => i(a.error);
|
|
174
222
|
});
|
|
175
223
|
}
|
|
176
|
-
function
|
|
177
|
-
return new Promise((
|
|
178
|
-
let
|
|
179
|
-
|
|
224
|
+
function p(e, t) {
|
|
225
|
+
return new Promise((n, r) => {
|
|
226
|
+
let i = u(e, t).getAll();
|
|
227
|
+
i.onsuccess = () => n(i.result), i.onerror = () => r(i.error);
|
|
180
228
|
});
|
|
181
229
|
}
|
|
182
|
-
var
|
|
230
|
+
var m = class {
|
|
183
231
|
constructor() {
|
|
184
232
|
this.db = null, this.hnsw = null, this.shardInfo = null, this.saveCounter = 0;
|
|
185
233
|
}
|
|
186
234
|
async initialize() {
|
|
187
|
-
this.db = await
|
|
235
|
+
this.db = await l("flaxia-vector-store", 1, (e) => {
|
|
188
236
|
e.objectStoreNames.contains("vectors") || e.createObjectStore("vectors", { keyPath: "docId" }), e.objectStoreNames.contains("hnsw-graph") || e.createObjectStore("hnsw-graph", { keyPath: "nodeId" }), e.objectStoreNames.contains("shard-info") || e.createObjectStore("shard-info", { keyPath: "key" });
|
|
189
|
-
}), this.hnsw = new
|
|
190
|
-
let
|
|
191
|
-
|
|
192
|
-
rangeStart:
|
|
193
|
-
rangeEnd:
|
|
237
|
+
}), this.hnsw = new n(1024, "cosine", 16, 200), await this.loadGraphFromDB();
|
|
238
|
+
let e = await d(this.db, "shard-info", "shard_range");
|
|
239
|
+
e && (this.shardInfo = {
|
|
240
|
+
rangeStart: e.rangeStart,
|
|
241
|
+
rangeEnd: e.rangeEnd
|
|
194
242
|
});
|
|
195
243
|
}
|
|
196
244
|
async assignShard(e, t) {
|
|
197
245
|
this.shardInfo = {
|
|
198
246
|
rangeStart: e,
|
|
199
247
|
rangeEnd: t
|
|
200
|
-
}, await
|
|
248
|
+
}, await f(this.db, "shard-info", {
|
|
201
249
|
key: "shard_range",
|
|
202
250
|
rangeStart: e,
|
|
203
251
|
rangeEnd: t,
|
|
@@ -207,15 +255,17 @@ var o = class {
|
|
|
207
255
|
}
|
|
208
256
|
async store(e) {
|
|
209
257
|
if (!this.shardInfo) throw Error("No shard assigned");
|
|
210
|
-
let t =
|
|
211
|
-
if (
|
|
212
|
-
|
|
258
|
+
let t = c(e.shardKey, this.shardInfo.rangeStart, this.shardInfo.rangeEnd), n = s(e.vector, "vector");
|
|
259
|
+
if (this.hnsw && n !== this.hnsw.dimensionsValue) throw Error(`vector-store: vector has ${n} dimensions but this index stores ${this.hnsw.dimensionsValue}`);
|
|
260
|
+
if (this.hnsw && this.hnsw.size() >= a) throw Error(`vector-store: node storage quota reached (${a} vectors); refusing to store more`);
|
|
261
|
+
let r = new Float32Array(e.vector);
|
|
262
|
+
return await f(this.db, "vectors", {
|
|
213
263
|
docId: e.docId,
|
|
214
|
-
vector:
|
|
264
|
+
vector: r,
|
|
215
265
|
metadata: e.metadata,
|
|
216
|
-
shardKey:
|
|
266
|
+
shardKey: String(t),
|
|
217
267
|
storedAt: Date.now()
|
|
218
|
-
}), this.hnsw.insert(e.docId,
|
|
268
|
+
}), this.hnsw.insert(e.docId, r), this.saveCounter++, this.saveCounter % 100 == 0 && await this.saveGraphSnapshot(), {
|
|
219
269
|
stored: !0,
|
|
220
270
|
nodeId: "",
|
|
221
271
|
totalVectors: this.hnsw.size()
|
|
@@ -223,9 +273,13 @@ var o = class {
|
|
|
223
273
|
}
|
|
224
274
|
async query(e) {
|
|
225
275
|
if (!this.hnsw) throw Error("HNSW not initialized");
|
|
226
|
-
let t =
|
|
276
|
+
let t = s(e.queryVector, "queryVector");
|
|
277
|
+
if (t !== this.hnsw.dimensionsValue) throw Error(`vector-store: queryVector has ${t} dimensions but this index stores ${this.hnsw.dimensionsValue}`);
|
|
278
|
+
let n = e.topK;
|
|
279
|
+
if (!Number.isInteger(n) || n <= 0 || n > 1e3) throw Error(`vector-store: topK must be an integer within 1..${o}, got ${JSON.stringify(n)}`);
|
|
280
|
+
let r = performance.now(), i = new Float32Array(e.queryVector);
|
|
227
281
|
return {
|
|
228
|
-
results: this.hnsw.search(
|
|
282
|
+
results: this.hnsw.search(i, n).map((e) => ({
|
|
229
283
|
docId: e.docId,
|
|
230
284
|
score: 1 - e.distance,
|
|
231
285
|
metadata: e.metadata || {
|
|
@@ -235,39 +289,40 @@ var o = class {
|
|
|
235
289
|
}
|
|
236
290
|
})),
|
|
237
291
|
nodeId: "",
|
|
238
|
-
searchDurationMs: Math.round(performance.now() -
|
|
292
|
+
searchDurationMs: Math.round(performance.now() - r)
|
|
239
293
|
};
|
|
240
294
|
}
|
|
241
295
|
async getVector(e) {
|
|
242
|
-
return
|
|
296
|
+
return d(this.db, "vectors", e);
|
|
243
297
|
}
|
|
244
298
|
async deleteVector(e) {
|
|
245
|
-
return new Promise((t,
|
|
246
|
-
let
|
|
247
|
-
|
|
299
|
+
return new Promise((t, n) => {
|
|
300
|
+
let r = u(this.db, "vectors", "readwrite").delete(e);
|
|
301
|
+
r.onsuccess = () => t(), r.onerror = () => n(r.error);
|
|
248
302
|
});
|
|
249
303
|
}
|
|
250
304
|
getVectorCount() {
|
|
251
305
|
return this.hnsw?.size() || 0;
|
|
252
306
|
}
|
|
253
307
|
async loadGraphFromDB() {
|
|
254
|
-
let e = await
|
|
308
|
+
let e = await p(this.db, "hnsw-graph");
|
|
255
309
|
if (e.length === 0) return;
|
|
310
|
+
if (e.length > a) throw Error(`vector-store: persisted graph holds ${e.length} nodes, above the ${a} quota`);
|
|
256
311
|
let t = /* @__PURE__ */ new Map(), n = [];
|
|
257
|
-
for (let
|
|
258
|
-
let e = await
|
|
259
|
-
e && t.set(
|
|
260
|
-
id:
|
|
261
|
-
docId:
|
|
262
|
-
level:
|
|
263
|
-
neighbors:
|
|
312
|
+
for (let r of e) {
|
|
313
|
+
let e = await d(this.db, "vectors", r.docId);
|
|
314
|
+
e && e.vector && t.set(r.nodeId, e.vector instanceof Float32Array ? e.vector : new Float32Array(e.vector)), n.push({
|
|
315
|
+
id: r.nodeId,
|
|
316
|
+
docId: r.docId,
|
|
317
|
+
level: r.level,
|
|
318
|
+
neighbors: r.neighbors
|
|
264
319
|
});
|
|
265
320
|
}
|
|
266
321
|
this.hnsw.importNodes(n, t);
|
|
267
322
|
}
|
|
268
323
|
async saveGraphSnapshot() {
|
|
269
324
|
let e = this.hnsw.exportNodes();
|
|
270
|
-
for (let [t, n] of e) await
|
|
325
|
+
for (let [t, n] of e) await f(this.db, "hnsw-graph", {
|
|
271
326
|
nodeId: t,
|
|
272
327
|
docId: n.docId,
|
|
273
328
|
level: n.level,
|
|
@@ -276,4 +331,4 @@ var o = class {
|
|
|
276
331
|
}
|
|
277
332
|
};
|
|
278
333
|
//#endregion
|
|
279
|
-
export {
|
|
334
|
+
export { m as t };
|