@fortemi/core 2026.7.3 → 2026.7.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/dist/aiwg-index-schema.d.ts +9 -0
- package/dist/aiwg-index-schema.js +724 -0
- package/dist/aiwg-index-schema.js.map +1 -0
- package/dist/aiwg-index.d.ts +32 -17
- package/dist/aiwg-index.js +15 -1463
- package/dist/aiwg-index.js.map +1 -1
- package/dist/index.d.ts +803 -96
- package/dist/index.js +2932 -223
- package/dist/index.js.map +1 -1
- package/package.json +19 -3
- package/tools/verify-fortemi-compatibility.mjs +82 -0
package/dist/index.js
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import { v7 } from 'uuid';
|
|
2
|
-
import { PGlite } from '@electric-sql/pglite';
|
|
3
|
-
import { vector } from '@electric-sql/pglite/vector';
|
|
4
2
|
import { sha256 } from '@noble/hashes/sha256';
|
|
3
|
+
import { blake3 } from '@noble/hashes/blake3';
|
|
5
4
|
import { bytesToHex } from '@noble/hashes/utils';
|
|
6
5
|
import { z } from 'zod';
|
|
7
6
|
import { gzipSync, gunzipSync } from 'fflate';
|
|
7
|
+
import Ajv2020 from 'ajv/dist/2020.js';
|
|
8
8
|
|
|
9
9
|
// src/uuid.ts
|
|
10
10
|
function generateId() {
|
|
@@ -125,6 +125,8 @@ var TypedEventBus = class {
|
|
|
125
125
|
this.wildcardListeners.clear();
|
|
126
126
|
}
|
|
127
127
|
};
|
|
128
|
+
|
|
129
|
+
// src/db.ts
|
|
128
130
|
function getDataDir(persistence, archiveName) {
|
|
129
131
|
switch (persistence) {
|
|
130
132
|
case "opfs":
|
|
@@ -137,6 +139,10 @@ function getDataDir(persistence, archiveName) {
|
|
|
137
139
|
}
|
|
138
140
|
async function createPGliteInstance(persistence, archiveName = "default", options = {}) {
|
|
139
141
|
const dataDir = getDataDir(persistence, archiveName);
|
|
142
|
+
const [{ PGlite }, { vector }] = await Promise.all([
|
|
143
|
+
import('@electric-sql/pglite'),
|
|
144
|
+
import('@electric-sql/pglite/vector')
|
|
145
|
+
]);
|
|
140
146
|
const pgliteOptions = {
|
|
141
147
|
database: "postgres",
|
|
142
148
|
// PGlite 0.4.x breaking change: explicit required
|
|
@@ -609,6 +615,134 @@ var migration0010 = {
|
|
|
609
615
|
`
|
|
610
616
|
};
|
|
611
617
|
|
|
618
|
+
// src/migrations/0011_embedding_member_metadata.ts
|
|
619
|
+
var migration0011 = {
|
|
620
|
+
version: 11,
|
|
621
|
+
name: "0011_embedding_member_metadata",
|
|
622
|
+
sql: `
|
|
623
|
+
ALTER TABLE embedding_set_member
|
|
624
|
+
ALTER COLUMN embedding_id DROP NOT NULL;
|
|
625
|
+
|
|
626
|
+
ALTER TABLE embedding_set_member
|
|
627
|
+
ADD COLUMN IF NOT EXISTS membership_type TEXT NOT NULL DEFAULT 'materialized';
|
|
628
|
+
|
|
629
|
+
ALTER TABLE embedding_set_member
|
|
630
|
+
ADD COLUMN IF NOT EXISTS added_at TIMESTAMPTZ NOT NULL DEFAULT now();
|
|
631
|
+
|
|
632
|
+
ALTER TABLE embedding_set_member
|
|
633
|
+
ADD COLUMN IF NOT EXISTS added_by TEXT;
|
|
634
|
+
`
|
|
635
|
+
};
|
|
636
|
+
|
|
637
|
+
// src/migrations/0012_embedding_configs.ts
|
|
638
|
+
var migration0012 = {
|
|
639
|
+
version: 12,
|
|
640
|
+
name: "0012_embedding_configs",
|
|
641
|
+
sql: `
|
|
642
|
+
CREATE TABLE IF NOT EXISTS embedding_config (
|
|
643
|
+
id TEXT PRIMARY KEY,
|
|
644
|
+
name TEXT NOT NULL,
|
|
645
|
+
description TEXT,
|
|
646
|
+
model TEXT NOT NULL,
|
|
647
|
+
dimension INTEGER NOT NULL,
|
|
648
|
+
chunk_size INTEGER NOT NULL,
|
|
649
|
+
chunk_overlap INTEGER NOT NULL,
|
|
650
|
+
is_default BOOLEAN NOT NULL DEFAULT false
|
|
651
|
+
);
|
|
652
|
+
|
|
653
|
+
CREATE INDEX IF NOT EXISTS idx_embedding_config_default ON embedding_config(is_default);
|
|
654
|
+
`
|
|
655
|
+
};
|
|
656
|
+
|
|
657
|
+
// src/migrations/0013_templates.ts
|
|
658
|
+
var migration0013 = {
|
|
659
|
+
version: 13,
|
|
660
|
+
name: "0013_templates",
|
|
661
|
+
sql: `
|
|
662
|
+
CREATE TABLE IF NOT EXISTS template (
|
|
663
|
+
id TEXT PRIMARY KEY,
|
|
664
|
+
name TEXT NOT NULL,
|
|
665
|
+
description TEXT,
|
|
666
|
+
content TEXT NOT NULL,
|
|
667
|
+
format TEXT NOT NULL DEFAULT 'markdown',
|
|
668
|
+
default_tags JSONB NOT NULL DEFAULT '[]',
|
|
669
|
+
collection_id TEXT,
|
|
670
|
+
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
|
671
|
+
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
|
672
|
+
);
|
|
673
|
+
|
|
674
|
+
CREATE INDEX IF NOT EXISTS idx_template_collection ON template(collection_id);
|
|
675
|
+
`
|
|
676
|
+
};
|
|
677
|
+
|
|
678
|
+
// src/migrations/0014_url_links.ts
|
|
679
|
+
var migration0014 = {
|
|
680
|
+
version: 14,
|
|
681
|
+
name: "0014_url_links",
|
|
682
|
+
sql: `
|
|
683
|
+
CREATE TABLE IF NOT EXISTS link_url_target (
|
|
684
|
+
id TEXT PRIMARY KEY,
|
|
685
|
+
source_note_id TEXT NOT NULL,
|
|
686
|
+
to_url TEXT NOT NULL,
|
|
687
|
+
link_type TEXT NOT NULL DEFAULT 'reference',
|
|
688
|
+
confidence REAL,
|
|
689
|
+
metadata_json JSONB,
|
|
690
|
+
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
|
691
|
+
updated_at TIMESTAMPTZ,
|
|
692
|
+
deleted_at TIMESTAMPTZ
|
|
693
|
+
);
|
|
694
|
+
|
|
695
|
+
CREATE INDEX IF NOT EXISTS idx_link_url_source ON link_url_target(source_note_id);
|
|
696
|
+
`
|
|
697
|
+
};
|
|
698
|
+
|
|
699
|
+
// src/migrations/0015_embedding_set_server_metadata.ts
|
|
700
|
+
var migration0015 = {
|
|
701
|
+
version: 15,
|
|
702
|
+
name: "0015_embedding_set_server_metadata",
|
|
703
|
+
sql: `
|
|
704
|
+
ALTER TABLE embedding_set
|
|
705
|
+
ADD COLUMN IF NOT EXISTS slug TEXT;
|
|
706
|
+
|
|
707
|
+
ALTER TABLE embedding_set
|
|
708
|
+
ADD COLUMN IF NOT EXISTS description TEXT;
|
|
709
|
+
|
|
710
|
+
ALTER TABLE embedding_set
|
|
711
|
+
ADD COLUMN IF NOT EXISTS document_count INTEGER;
|
|
712
|
+
|
|
713
|
+
ALTER TABLE embedding_set
|
|
714
|
+
ADD COLUMN IF NOT EXISTS embedding_count INTEGER;
|
|
715
|
+
|
|
716
|
+
ALTER TABLE embedding_set
|
|
717
|
+
ADD COLUMN IF NOT EXISTS is_system BOOLEAN NOT NULL DEFAULT FALSE;
|
|
718
|
+
|
|
719
|
+
ALTER TABLE embedding_set
|
|
720
|
+
ADD COLUMN IF NOT EXISTS keywords_json JSONB;
|
|
721
|
+
`
|
|
722
|
+
};
|
|
723
|
+
|
|
724
|
+
// src/migrations/0016_embedding_server_metadata.ts
|
|
725
|
+
var migration0016 = {
|
|
726
|
+
version: 16,
|
|
727
|
+
name: "0016_embedding_server_metadata",
|
|
728
|
+
sql: `
|
|
729
|
+
ALTER TABLE embedding
|
|
730
|
+
ADD COLUMN IF NOT EXISTS chunk_index INTEGER NOT NULL DEFAULT 0;
|
|
731
|
+
|
|
732
|
+
ALTER TABLE embedding
|
|
733
|
+
ADD COLUMN IF NOT EXISTS text TEXT NOT NULL DEFAULT '';
|
|
734
|
+
|
|
735
|
+
ALTER TABLE embedding
|
|
736
|
+
ADD COLUMN IF NOT EXISTS model TEXT;
|
|
737
|
+
|
|
738
|
+
ALTER TABLE embedding
|
|
739
|
+
DROP CONSTRAINT IF EXISTS embedding_note_id_embedding_set_id_key;
|
|
740
|
+
|
|
741
|
+
CREATE UNIQUE INDEX IF NOT EXISTS idx_embedding_note_set_chunk
|
|
742
|
+
ON embedding(note_id, embedding_set_id, chunk_index);
|
|
743
|
+
`
|
|
744
|
+
};
|
|
745
|
+
|
|
612
746
|
// src/migrations/index.ts
|
|
613
747
|
var allMigrations = [
|
|
614
748
|
migration0001,
|
|
@@ -620,7 +754,13 @@ var allMigrations = [
|
|
|
620
754
|
migration0007,
|
|
621
755
|
migration0008,
|
|
622
756
|
migration0009,
|
|
623
|
-
migration0010
|
|
757
|
+
migration0010,
|
|
758
|
+
migration0011,
|
|
759
|
+
migration0012,
|
|
760
|
+
migration0013,
|
|
761
|
+
migration0014,
|
|
762
|
+
migration0015,
|
|
763
|
+
migration0016
|
|
624
764
|
];
|
|
625
765
|
|
|
626
766
|
// src/data-archive.ts
|
|
@@ -730,6 +870,9 @@ function computeHash(data) {
|
|
|
730
870
|
const digest = sha256(data);
|
|
731
871
|
return `sha256:${bytesToHex(digest)}`;
|
|
732
872
|
}
|
|
873
|
+
function computeBlobHash(data) {
|
|
874
|
+
return `blake3:${bytesToHex(blake3(data))}`;
|
|
875
|
+
}
|
|
733
876
|
|
|
734
877
|
// src/repositories/notes-repository.ts
|
|
735
878
|
var NotesRepository = class {
|
|
@@ -1288,11 +1431,11 @@ var EmbeddingSetsRepository = class {
|
|
|
1288
1431
|
[input.note_id, input.embedding_set_id]
|
|
1289
1432
|
);
|
|
1290
1433
|
const embeddingId = input.id ?? generateId();
|
|
1291
|
-
const
|
|
1434
|
+
const vector = `[${input.vector.join(",")}]`;
|
|
1292
1435
|
await this.db.query(
|
|
1293
1436
|
`INSERT INTO embedding (id, note_id, embedding_set_id, vector)
|
|
1294
1437
|
VALUES ($1, $2, $3, $4::vector)`,
|
|
1295
|
-
[embeddingId, input.note_id, input.embedding_set_id,
|
|
1438
|
+
[embeddingId, input.note_id, input.embedding_set_id, vector]
|
|
1296
1439
|
);
|
|
1297
1440
|
await this.db.query(
|
|
1298
1441
|
`INSERT INTO embedding_set_member (embedding_set_id, note_id, embedding_id)
|
|
@@ -1454,7 +1597,7 @@ var EmbeddingSetsRepository = class {
|
|
|
1454
1597
|
if (criteria.conceptIds && criteria.conceptIds.length > 0) {
|
|
1455
1598
|
throw new Error("Unsupported virtual embedding-set criteria field: conceptIds");
|
|
1456
1599
|
}
|
|
1457
|
-
const conditions = ["e.embedding_set_id = $1"];
|
|
1600
|
+
const conditions = ["e.embedding_set_id = $1", "n.deleted_at IS NULL"];
|
|
1458
1601
|
const params = [source.baseSetId];
|
|
1459
1602
|
let idx = 2;
|
|
1460
1603
|
if (criteria.noteIds?.length) {
|
|
@@ -1657,7 +1800,7 @@ var ATTACHMENT_TEXT_JOIN2 = `
|
|
|
1657
1800
|
WHERE deleted_at IS NULL
|
|
1658
1801
|
GROUP BY note_id
|
|
1659
1802
|
) ax ON ax.note_id = n.id`;
|
|
1660
|
-
var COMBINED_TEXT_SQL = `(coalesce(c.content, '') || ' ' || coalesce(ax.extracted_text, ''))`;
|
|
1803
|
+
var COMBINED_TEXT_SQL = `trim(both from (coalesce(c.content, '') || ' ' || coalesce(ax.extracted_text, '')))`;
|
|
1661
1804
|
var COMBINED_TEXT_VECTOR_SQL2 = `to_tsvector('english', ${COMBINED_TEXT_SQL})`;
|
|
1662
1805
|
var SearchRepository = class {
|
|
1663
1806
|
constructor(db, semanticAvailable = false) {
|
|
@@ -2285,11 +2428,25 @@ function linkToBackend(link) {
|
|
|
2285
2428
|
createdAt: toIso(link.created_at)
|
|
2286
2429
|
};
|
|
2287
2430
|
}
|
|
2431
|
+
function urlLinkToBackend(link) {
|
|
2432
|
+
const metadata = typeof link.metadata_json === "string" ? JSON.parse(link.metadata_json) : link.metadata_json ?? void 0;
|
|
2433
|
+
return {
|
|
2434
|
+
id: link.id,
|
|
2435
|
+
fromNoteId: link.source_note_id,
|
|
2436
|
+
toNoteId: null,
|
|
2437
|
+
toUrl: link.to_url,
|
|
2438
|
+
kind: link.link_type,
|
|
2439
|
+
score: link.confidence,
|
|
2440
|
+
createdAt: toIso(link.created_at),
|
|
2441
|
+
...metadata ? { metadata } : {}
|
|
2442
|
+
};
|
|
2443
|
+
}
|
|
2288
2444
|
function shardLinkToBackend(link) {
|
|
2289
2445
|
return {
|
|
2290
2446
|
id: link.id,
|
|
2291
2447
|
fromNoteId: link.from_note_id,
|
|
2292
2448
|
toNoteId: link.to_note_id,
|
|
2449
|
+
toUrl: link.to_url,
|
|
2293
2450
|
kind: link.kind,
|
|
2294
2451
|
score: link.score,
|
|
2295
2452
|
createdAt: link.created_at,
|
|
@@ -2335,7 +2492,8 @@ function remoteLinkToBackend(link) {
|
|
|
2335
2492
|
return {
|
|
2336
2493
|
id: link.id,
|
|
2337
2494
|
fromNoteId: link.fromNoteId ?? link.from_note_id ?? link.source_note_id ?? "",
|
|
2338
|
-
toNoteId: link.toNoteId ?? link.to_note_id ?? link.target_note_id ??
|
|
2495
|
+
toNoteId: link.toNoteId ?? link.to_note_id ?? link.target_note_id ?? null,
|
|
2496
|
+
toUrl: link.toUrl ?? link.to_url ?? null,
|
|
2339
2497
|
kind: link.kind ?? link.link_type ?? "",
|
|
2340
2498
|
score: link.score ?? link.confidence ?? null,
|
|
2341
2499
|
createdAt: link.createdAt ?? link.created_at ?? "",
|
|
@@ -2357,7 +2515,15 @@ function createPGliteBackend(db, options = {}) {
|
|
|
2357
2515
|
const links = new LinksRepository(db);
|
|
2358
2516
|
async function linksOf(id) {
|
|
2359
2517
|
const result = await links.listForNote(id);
|
|
2360
|
-
|
|
2518
|
+
const urlLinks = await db.query(
|
|
2519
|
+
`SELECT * FROM link_url_target WHERE source_note_id = $1 AND deleted_at IS NULL ORDER BY created_at`,
|
|
2520
|
+
[id]
|
|
2521
|
+
);
|
|
2522
|
+
return [
|
|
2523
|
+
...result.outbound.map(linkToBackend),
|
|
2524
|
+
...result.inbound.map(linkToBackend),
|
|
2525
|
+
...urlLinks.rows.map(urlLinkToBackend)
|
|
2526
|
+
];
|
|
2361
2527
|
}
|
|
2362
2528
|
async function conceptsOf(id) {
|
|
2363
2529
|
const result = await db.query(
|
|
@@ -2988,6 +3154,20 @@ var MigrationRunner = class {
|
|
|
2988
3154
|
|
|
2989
3155
|
// src/archive-manager.ts
|
|
2990
3156
|
var ArchiveManager = class {
|
|
3157
|
+
/**
|
|
3158
|
+
* Persistence is a PLUGGABLE, opt-in backend (issue #261). Pass a
|
|
3159
|
+
* `StorageBackendFactory` to run against any `StorageBackend`
|
|
3160
|
+
* implementation; PGlite is merely the built-in default, selected only when
|
|
3161
|
+
* the convenience `PersistenceMode` string form is used. Because
|
|
3162
|
+
* `defaultStorageBackendFactory` reaches PGlite through the lazily-imported
|
|
3163
|
+
* `createPGliteInstance` (see `db.ts`), a consumer that supplies its own
|
|
3164
|
+
* factory — or never opens an archive at all — never pulls the PGlite WASM
|
|
3165
|
+
* engine into its bundle.
|
|
3166
|
+
*
|
|
3167
|
+
* @param persistenceOrFactory `PersistenceMode` string (uses the built-in
|
|
3168
|
+
* PGlite backend) OR a custom `StorageBackendFactory`.
|
|
3169
|
+
* @param persistenceOverride persistence hint passed to a custom factory.
|
|
3170
|
+
*/
|
|
2991
3171
|
constructor(persistenceOrFactory, events, persistenceOverride) {
|
|
2992
3172
|
this.events = events;
|
|
2993
3173
|
if (typeof persistenceOrFactory === "string") {
|
|
@@ -3096,6 +3276,205 @@ function createFortemi(config) {
|
|
|
3096
3276
|
};
|
|
3097
3277
|
}
|
|
3098
3278
|
|
|
3279
|
+
// src/server-compatibility.ts
|
|
3280
|
+
var FORTEMI_COMPATIBILITY_PATH = "/api/v1/system/compatibility";
|
|
3281
|
+
var FORTEMI_COMPATIBILITY_STATES = [
|
|
3282
|
+
"available",
|
|
3283
|
+
"degraded",
|
|
3284
|
+
"preview",
|
|
3285
|
+
"unavailable",
|
|
3286
|
+
"unknown"
|
|
3287
|
+
];
|
|
3288
|
+
var FORTEMI_REQUIRED_COMPATIBILITY_CAPABILITIES = [
|
|
3289
|
+
"core_notes",
|
|
3290
|
+
"search",
|
|
3291
|
+
"jobs",
|
|
3292
|
+
"realtime_activity",
|
|
3293
|
+
"hosted_auth",
|
|
3294
|
+
"premium_components",
|
|
3295
|
+
"backoffice_api",
|
|
3296
|
+
"audit_posture",
|
|
3297
|
+
"quota_status",
|
|
3298
|
+
"kms_status",
|
|
3299
|
+
"mcp_scope_gate"
|
|
3300
|
+
];
|
|
3301
|
+
function isRecord(value) {
|
|
3302
|
+
return !!value && typeof value === "object" && !Array.isArray(value);
|
|
3303
|
+
}
|
|
3304
|
+
function isSupportedState(value) {
|
|
3305
|
+
return typeof value === "string" && FORTEMI_COMPATIBILITY_STATES.includes(value);
|
|
3306
|
+
}
|
|
3307
|
+
function requireRecord(parent, key, errors) {
|
|
3308
|
+
const value = parent[key];
|
|
3309
|
+
if (!isRecord(value)) {
|
|
3310
|
+
errors.push(`${key} must be an object`);
|
|
3311
|
+
return null;
|
|
3312
|
+
}
|
|
3313
|
+
return value;
|
|
3314
|
+
}
|
|
3315
|
+
function requireString(parent, key, path, errors) {
|
|
3316
|
+
const value = parent[key];
|
|
3317
|
+
if (typeof value !== "string" || value.length === 0) {
|
|
3318
|
+
errors.push(`${path}.${key} must be a non-empty string`);
|
|
3319
|
+
return null;
|
|
3320
|
+
}
|
|
3321
|
+
return value;
|
|
3322
|
+
}
|
|
3323
|
+
function requireBoolean(parent, key, path, errors) {
|
|
3324
|
+
const value = parent[key];
|
|
3325
|
+
if (typeof value !== "boolean") {
|
|
3326
|
+
errors.push(`${path}.${key} must be a boolean`);
|
|
3327
|
+
return null;
|
|
3328
|
+
}
|
|
3329
|
+
return value;
|
|
3330
|
+
}
|
|
3331
|
+
function fortemiCompatibilityUrl(baseUrl = "http://localhost:3000") {
|
|
3332
|
+
const trimmed = baseUrl.trim().replace(/\/+$/, "");
|
|
3333
|
+
if (trimmed.endsWith(FORTEMI_COMPATIBILITY_PATH)) return trimmed;
|
|
3334
|
+
return `${trimmed}${FORTEMI_COMPATIBILITY_PATH}`;
|
|
3335
|
+
}
|
|
3336
|
+
function validateFortemiCompatibilityResponse(raw) {
|
|
3337
|
+
const errors = [];
|
|
3338
|
+
const warnings = [];
|
|
3339
|
+
if (!isRecord(raw)) {
|
|
3340
|
+
return {
|
|
3341
|
+
ok: false,
|
|
3342
|
+
errors: ["response must be a JSON object"],
|
|
3343
|
+
warnings
|
|
3344
|
+
};
|
|
3345
|
+
}
|
|
3346
|
+
if (raw.schema_version !== 1) {
|
|
3347
|
+
errors.push("schema_version must be 1");
|
|
3348
|
+
}
|
|
3349
|
+
requireString(raw, "contract_revision", "response", errors);
|
|
3350
|
+
const api = requireRecord(raw, "api", errors);
|
|
3351
|
+
if (api) {
|
|
3352
|
+
const name = requireString(api, "name", "api", errors);
|
|
3353
|
+
if (name && name !== "fortemi") {
|
|
3354
|
+
errors.push("api.name must be fortemi");
|
|
3355
|
+
}
|
|
3356
|
+
requireString(api, "version", "api", errors);
|
|
3357
|
+
requireString(api, "minimum_hotm_enterprise_client", "api", errors);
|
|
3358
|
+
requireBoolean(api, "git_sha_present", "api", errors);
|
|
3359
|
+
requireBoolean(api, "build_date_present", "api", errors);
|
|
3360
|
+
}
|
|
3361
|
+
const deployment = requireRecord(raw, "deployment", errors);
|
|
3362
|
+
if (deployment) {
|
|
3363
|
+
requireString(deployment, "mode", "deployment", errors);
|
|
3364
|
+
requireString(deployment, "edition", "deployment", errors);
|
|
3365
|
+
requireBoolean(deployment, "hosted_multi_tenant_ready", "deployment", errors);
|
|
3366
|
+
}
|
|
3367
|
+
const auth = requireRecord(raw, "auth", errors);
|
|
3368
|
+
if (auth) {
|
|
3369
|
+
requireBoolean(auth, "required", "auth", errors);
|
|
3370
|
+
requireString(auth, "mode", "auth", errors);
|
|
3371
|
+
requireBoolean(auth, "oauth_issuer_configured", "auth", errors);
|
|
3372
|
+
requireBoolean(auth, "tenant_context_available", "auth", errors);
|
|
3373
|
+
}
|
|
3374
|
+
const capabilities = requireRecord(raw, "capabilities", errors);
|
|
3375
|
+
if (capabilities) {
|
|
3376
|
+
for (const key of FORTEMI_REQUIRED_COMPATIBILITY_CAPABILITIES) {
|
|
3377
|
+
if (!isRecord(capabilities[key])) {
|
|
3378
|
+
errors.push(`capabilities.${key} must be present as an object`);
|
|
3379
|
+
}
|
|
3380
|
+
}
|
|
3381
|
+
for (const [key, value] of Object.entries(capabilities)) {
|
|
3382
|
+
if (!isRecord(value)) {
|
|
3383
|
+
errors.push(`capabilities.${key} must be an object`);
|
|
3384
|
+
continue;
|
|
3385
|
+
}
|
|
3386
|
+
if (!isSupportedState(value.state)) {
|
|
3387
|
+
errors.push(`capabilities.${key}.state must be one of ${FORTEMI_COMPATIBILITY_STATES.join(", ")}`);
|
|
3388
|
+
}
|
|
3389
|
+
if ("reason_code" in value && typeof value.reason_code !== "string") {
|
|
3390
|
+
errors.push(`capabilities.${key}.reason_code must be a string when present`);
|
|
3391
|
+
}
|
|
3392
|
+
if (value.state !== "available" && !value.reason_code) {
|
|
3393
|
+
warnings.push(`capabilities.${key} is ${String(value.state)} without reason_code`);
|
|
3394
|
+
}
|
|
3395
|
+
}
|
|
3396
|
+
}
|
|
3397
|
+
const links = requireRecord(raw, "links", errors);
|
|
3398
|
+
if (links) {
|
|
3399
|
+
requireString(links, "openapi", "links", errors);
|
|
3400
|
+
requireString(links, "asyncapi", "links", errors);
|
|
3401
|
+
requireString(links, "health", "links", errors);
|
|
3402
|
+
requireString(links, "streaming_health", "links", errors);
|
|
3403
|
+
}
|
|
3404
|
+
return {
|
|
3405
|
+
ok: errors.length === 0,
|
|
3406
|
+
errors,
|
|
3407
|
+
warnings,
|
|
3408
|
+
response: errors.length === 0 ? raw : void 0
|
|
3409
|
+
};
|
|
3410
|
+
}
|
|
3411
|
+
function formatFortemiCompatibilitySummary(response) {
|
|
3412
|
+
const capabilitySummary = Object.entries(response.capabilities).map(([key, value]) => `${key}=${value.state}`).sort().join(", ");
|
|
3413
|
+
return [
|
|
3414
|
+
`Fortemi compatibility ${response.contract_revision}`,
|
|
3415
|
+
`api=${response.api.name}@${response.api.version}`,
|
|
3416
|
+
`deployment=${response.deployment.mode}/${response.deployment.edition}`,
|
|
3417
|
+
`auth=${response.auth.mode}`,
|
|
3418
|
+
`capabilities: ${capabilitySummary}`
|
|
3419
|
+
].join("\n");
|
|
3420
|
+
}
|
|
3421
|
+
async function fetchAndValidateFortemiCompatibility(options = {}) {
|
|
3422
|
+
const fetchImpl = options.fetchImpl ?? globalThis.fetch;
|
|
3423
|
+
if (!fetchImpl) {
|
|
3424
|
+
return {
|
|
3425
|
+
ok: false,
|
|
3426
|
+
url: fortemiCompatibilityUrl(options.baseUrl),
|
|
3427
|
+
errors: ["fetch is not available in this runtime"],
|
|
3428
|
+
warnings: []
|
|
3429
|
+
};
|
|
3430
|
+
}
|
|
3431
|
+
const url = fortemiCompatibilityUrl(options.baseUrl);
|
|
3432
|
+
const controller = new AbortController();
|
|
3433
|
+
const timeout = setTimeout(() => controller.abort(), options.timeoutMs ?? 5e3);
|
|
3434
|
+
try {
|
|
3435
|
+
const response = await fetchImpl(url, {
|
|
3436
|
+
method: "GET",
|
|
3437
|
+
headers: { accept: "application/json" },
|
|
3438
|
+
signal: controller.signal
|
|
3439
|
+
});
|
|
3440
|
+
const status = response.status;
|
|
3441
|
+
const contentType = response.headers.get("content-type") ?? "";
|
|
3442
|
+
if (!response.ok) {
|
|
3443
|
+
return {
|
|
3444
|
+
ok: false,
|
|
3445
|
+
url,
|
|
3446
|
+
status,
|
|
3447
|
+
errors: [`Fortemi compatibility endpoint returned HTTP ${status}`],
|
|
3448
|
+
warnings: []
|
|
3449
|
+
};
|
|
3450
|
+
}
|
|
3451
|
+
if (!contentType.toLowerCase().includes("application/json")) {
|
|
3452
|
+
return {
|
|
3453
|
+
ok: false,
|
|
3454
|
+
url,
|
|
3455
|
+
status,
|
|
3456
|
+
errors: [`Fortemi compatibility endpoint must return JSON; got ${contentType || "unknown content-type"}`],
|
|
3457
|
+
warnings: []
|
|
3458
|
+
};
|
|
3459
|
+
}
|
|
3460
|
+
return {
|
|
3461
|
+
...validateFortemiCompatibilityResponse(await response.json()),
|
|
3462
|
+
url,
|
|
3463
|
+
status
|
|
3464
|
+
};
|
|
3465
|
+
} catch (error) {
|
|
3466
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
3467
|
+
return {
|
|
3468
|
+
ok: false,
|
|
3469
|
+
url,
|
|
3470
|
+
errors: [`Fortemi compatibility endpoint is unreachable: ${message}`],
|
|
3471
|
+
warnings: []
|
|
3472
|
+
};
|
|
3473
|
+
} finally {
|
|
3474
|
+
clearTimeout(timeout);
|
|
3475
|
+
}
|
|
3476
|
+
}
|
|
3477
|
+
|
|
3099
3478
|
// src/service-worker/register.ts
|
|
3100
3479
|
async function registerServiceWorker(swUrl = "/sw.js") {
|
|
3101
3480
|
if (!("serviceWorker" in navigator)) {
|
|
@@ -3862,6 +4241,7 @@ async function getNoteTextWithExtractedAttachments(db, noteId) {
|
|
|
3862
4241
|
' ORDER BY a.position, a.created_at)
|
|
3863
4242
|
FILTER (WHERE a.extracted_text IS NOT NULL AND a.extracted_text <> ''), '') as extracted_text
|
|
3864
4243
|
FROM note_revised_current c
|
|
4244
|
+
JOIN note n ON n.id = c.note_id AND n.deleted_at IS NULL
|
|
3865
4245
|
LEFT JOIN attachment a ON a.note_id = c.note_id AND a.deleted_at IS NULL
|
|
3866
4246
|
WHERE c.note_id = $1
|
|
3867
4247
|
GROUP BY c.content`,
|
|
@@ -4159,7 +4539,7 @@ function conceptTaggingHandler(job, db) {
|
|
|
4159
4539
|
const llmFn2 = getLlmFunction();
|
|
4160
4540
|
if (!llmFn2) return { skipped: true, reason: "no LLM function registered" };
|
|
4161
4541
|
const noteText = await getNoteTextWithExtractedAttachments(db, job.note_id);
|
|
4162
|
-
if (!noteText)
|
|
4542
|
+
if (!noteText) return { skipped: true, reason: "note missing, deleted, or has no content" };
|
|
4163
4543
|
const content = noteText.combined;
|
|
4164
4544
|
const prompt = `Task: Extract 3-5 topic tags from the text below.
|
|
4165
4545
|
Rules:
|
|
@@ -4197,20 +4577,25 @@ Tags:`;
|
|
|
4197
4577
|
function linkingHandler(job, db) {
|
|
4198
4578
|
return (async () => {
|
|
4199
4579
|
const embResult = await db.query(
|
|
4200
|
-
`SELECT id FROM embedding
|
|
4580
|
+
`SELECT e.id FROM embedding e
|
|
4581
|
+
JOIN note n ON n.id = e.note_id AND n.deleted_at IS NULL
|
|
4582
|
+
WHERE e.note_id = $1 LIMIT 1`,
|
|
4201
4583
|
[job.note_id]
|
|
4202
4584
|
);
|
|
4203
4585
|
if (embResult.rows.length === 0) {
|
|
4204
4586
|
return { skipped: true, reason: "no embeddings for this note yet" };
|
|
4205
4587
|
}
|
|
4206
4588
|
const vecResult = await db.query(
|
|
4207
|
-
`SELECT vector::text FROM embedding
|
|
4589
|
+
`SELECT e.vector::text FROM embedding e
|
|
4590
|
+
JOIN note n ON n.id = e.note_id AND n.deleted_at IS NULL
|
|
4591
|
+
WHERE e.note_id = $1 LIMIT 1`,
|
|
4208
4592
|
[job.note_id]
|
|
4209
4593
|
);
|
|
4210
4594
|
if (vecResult.rows.length === 0) return { skipped: true, reason: "no vector found" };
|
|
4211
4595
|
const similar = await db.query(
|
|
4212
4596
|
`SELECT e.note_id, e.vector <=> (SELECT vector FROM embedding WHERE note_id = $1 LIMIT 1) as distance
|
|
4213
4597
|
FROM embedding e
|
|
4598
|
+
JOIN note n ON n.id = e.note_id AND n.deleted_at IS NULL
|
|
4214
4599
|
WHERE e.note_id != $1
|
|
4215
4600
|
ORDER BY distance ASC
|
|
4216
4601
|
LIMIT 5`,
|
|
@@ -5027,14 +5412,16 @@ var AttachmentsRepository = class {
|
|
|
5027
5412
|
/**
|
|
5028
5413
|
* Attach a binary file to a note.
|
|
5029
5414
|
*
|
|
5030
|
-
* If a blob with the same
|
|
5415
|
+
* If a blob with the same BLAKE3 content hash already exists, the existing
|
|
5031
5416
|
* blob row is reused (deduplication). Otherwise a new blob row is inserted
|
|
5032
|
-
* and the raw bytes are written to the BlobStore.
|
|
5417
|
+
* and the raw bytes are written to the BlobStore. The BLAKE3 `content_hash`
|
|
5418
|
+
* (`blake3:<hex>`) matches the server convention and is the key used by the
|
|
5419
|
+
* portable Knowledge-Shard byte sidecar.
|
|
5033
5420
|
*
|
|
5034
5421
|
* Returns the newly created AttachmentRow.
|
|
5035
5422
|
*/
|
|
5036
5423
|
async attach(input) {
|
|
5037
|
-
const contentHash =
|
|
5424
|
+
const contentHash = computeBlobHash(input.data);
|
|
5038
5425
|
const sizeBytes = input.data.length;
|
|
5039
5426
|
let blobId;
|
|
5040
5427
|
const existing = await this.db.query(
|
|
@@ -5148,6 +5535,10 @@ async function manageAttachments(db, blobStore, rawInput) {
|
|
|
5148
5535
|
extractedText: input.extracted_text,
|
|
5149
5536
|
displayName: input.display_name
|
|
5150
5537
|
});
|
|
5538
|
+
if (input.extracted_text) {
|
|
5539
|
+
await enqueueJob(db, { noteId: input.note_id, jobType: "embedding" });
|
|
5540
|
+
await enqueueJob(db, { noteId: input.note_id, jobType: "concept_tagging" });
|
|
5541
|
+
}
|
|
5151
5542
|
return { action: "attach", attachment, size_bytes: data.length };
|
|
5152
5543
|
}
|
|
5153
5544
|
case "list": {
|
|
@@ -5171,7 +5562,12 @@ async function manageAttachments(db, blobStore, rawInput) {
|
|
|
5171
5562
|
}
|
|
5172
5563
|
case "delete": {
|
|
5173
5564
|
if (!input.attachment_id) throw new Error("attachment_id required for delete");
|
|
5565
|
+
const attachment = await repo.get(input.attachment_id);
|
|
5174
5566
|
await repo.delete(input.attachment_id);
|
|
5567
|
+
if (attachment.extracted_text) {
|
|
5568
|
+
await enqueueJob(db, { noteId: attachment.note_id, jobType: "embedding" });
|
|
5569
|
+
await enqueueJob(db, { noteId: attachment.note_id, jobType: "concept_tagging" });
|
|
5570
|
+
}
|
|
5175
5571
|
return { action: "delete", attachment_id: input.attachment_id };
|
|
5176
5572
|
}
|
|
5177
5573
|
}
|
|
@@ -5398,17 +5794,17 @@ async function embeddingGenerationHandler(job, db) {
|
|
|
5398
5794
|
const fn = embedFn;
|
|
5399
5795
|
if (!fn) return { skipped: true, reason: "no embed function registered" };
|
|
5400
5796
|
const noteText = await getNoteTextWithExtractedAttachments(db, job.note_id);
|
|
5401
|
-
if (!noteText)
|
|
5797
|
+
if (!noteText) return { skipped: true, reason: "note missing, deleted, or has no content" };
|
|
5402
5798
|
const content = noteText.combined;
|
|
5403
5799
|
const chunks = chunkText(content);
|
|
5404
5800
|
const embeddings = await fn(chunks);
|
|
5405
|
-
const
|
|
5801
|
+
const vector = averageEmbeddings(embeddings);
|
|
5406
5802
|
const embeddingSets = new EmbeddingSetsRepository(db);
|
|
5407
5803
|
const set = await embeddingSets.ensureDefault();
|
|
5408
5804
|
await embeddingSets.putEmbedding({
|
|
5409
5805
|
note_id: job.note_id,
|
|
5410
5806
|
embedding_set_id: set.id,
|
|
5411
|
-
vector
|
|
5807
|
+
vector
|
|
5412
5808
|
});
|
|
5413
5809
|
return { chunks: chunks.length, embeddings: embeddings.length, setId: set.id };
|
|
5414
5810
|
}
|
|
@@ -5827,13 +6223,13 @@ var OpenAICompatibleProvider = class {
|
|
|
5827
6223
|
throw new Error("Streaming not supported: response body is null");
|
|
5828
6224
|
}
|
|
5829
6225
|
const reader = response.body.getReader();
|
|
5830
|
-
const
|
|
6226
|
+
const decoder6 = new TextDecoder();
|
|
5831
6227
|
let buffer = "";
|
|
5832
6228
|
try {
|
|
5833
6229
|
while (true) {
|
|
5834
6230
|
const { done, value } = await reader.read();
|
|
5835
6231
|
if (done) break;
|
|
5836
|
-
buffer +=
|
|
6232
|
+
buffer += decoder6.decode(value, { stream: true });
|
|
5837
6233
|
const lines = buffer.split("\n");
|
|
5838
6234
|
buffer = lines.pop() ?? "";
|
|
5839
6235
|
for (const line of lines) {
|
|
@@ -6453,6 +6849,23 @@ function createCspReportHandler(onReport) {
|
|
|
6453
6849
|
// src/shard/types.ts
|
|
6454
6850
|
var CURRENT_SHARD_VERSION = "1.0.0";
|
|
6455
6851
|
var SHARD_FORMAT = "matric-shard";
|
|
6852
|
+
function parseVersion(value) {
|
|
6853
|
+
return value.split(".").map((segment) => {
|
|
6854
|
+
const match = segment.match(/^\d+/);
|
|
6855
|
+
return match ? Number.parseInt(match[0], 10) : 0;
|
|
6856
|
+
});
|
|
6857
|
+
}
|
|
6858
|
+
function compareShardVersions(left, right) {
|
|
6859
|
+
const leftParts = parseVersion(left);
|
|
6860
|
+
const rightParts = parseVersion(right);
|
|
6861
|
+
const length = Math.max(leftParts.length, rightParts.length);
|
|
6862
|
+
for (let index = 0; index < length; index += 1) {
|
|
6863
|
+
const leftValue = leftParts[index] ?? 0;
|
|
6864
|
+
const rightValue = rightParts[index] ?? 0;
|
|
6865
|
+
if (leftValue !== rightValue) return leftValue > rightValue ? 1 : -1;
|
|
6866
|
+
}
|
|
6867
|
+
return 0;
|
|
6868
|
+
}
|
|
6456
6869
|
var BLOCK_SIZE = 512;
|
|
6457
6870
|
var USTAR_MAGIC = "ustar\x0000";
|
|
6458
6871
|
function writeString(buf, offset, str, len) {
|
|
@@ -6542,8 +6955,14 @@ function decodeTar(tarData) {
|
|
|
6542
6955
|
}
|
|
6543
6956
|
return files;
|
|
6544
6957
|
}
|
|
6545
|
-
function packTarGz(files) {
|
|
6958
|
+
function packTarGz(files, opts) {
|
|
6546
6959
|
const tarData = encodeTar(files);
|
|
6960
|
+
const cap = opts?.maxDecompressedBytes ?? DEFAULT_MAX_DECOMPRESSED_BYTES;
|
|
6961
|
+
if (tarData.byteLength > cap) {
|
|
6962
|
+
throw new Error(
|
|
6963
|
+
"Refusing to create archive: uncompressed size " + tarData.byteLength + " exceeds cap " + cap + " bytes"
|
|
6964
|
+
);
|
|
6965
|
+
}
|
|
6547
6966
|
return gzipSync(tarData);
|
|
6548
6967
|
}
|
|
6549
6968
|
var DEFAULT_MAX_DECOMPRESSED_BYTES = 256 * 1024 * 1024;
|
|
@@ -6599,7 +7018,8 @@ function noteToShard(note) {
|
|
|
6599
7018
|
title: note.title,
|
|
6600
7019
|
original_content: note.original_content,
|
|
6601
7020
|
revised_content: note.revised_content,
|
|
6602
|
-
|
|
7021
|
+
collection_id: note.collection_id ?? null,
|
|
7022
|
+
...note.attachments?.length ? { attachments: note.attachments } : {},
|
|
6603
7023
|
format: note.format,
|
|
6604
7024
|
source: note.source,
|
|
6605
7025
|
starred: note.is_starred,
|
|
@@ -6611,6 +7031,7 @@ function noteToShard(note) {
|
|
|
6611
7031
|
};
|
|
6612
7032
|
}
|
|
6613
7033
|
function noteFromShard(shard) {
|
|
7034
|
+
const attachments = shard.attachments ?? shard.binary_sources;
|
|
6614
7035
|
return {
|
|
6615
7036
|
id: shard.id,
|
|
6616
7037
|
title: shard.title,
|
|
@@ -6620,7 +7041,8 @@ function noteFromShard(shard) {
|
|
|
6620
7041
|
is_archived: shard.archived,
|
|
6621
7042
|
original_content: shard.original_content,
|
|
6622
7043
|
revised_content: shard.revised_content,
|
|
6623
|
-
|
|
7044
|
+
collection_id: shard.collection_id ?? null,
|
|
7045
|
+
attachments,
|
|
6624
7046
|
tags: shard.tags,
|
|
6625
7047
|
created_at: shard.created_at,
|
|
6626
7048
|
updated_at: shard.updated_at,
|
|
@@ -6632,9 +7054,24 @@ function linkToShard(link) {
|
|
|
6632
7054
|
id: link.id,
|
|
6633
7055
|
from_note_id: link.source_note_id,
|
|
6634
7056
|
to_note_id: link.target_note_id,
|
|
7057
|
+
to_url: null,
|
|
7058
|
+
kind: link.link_type,
|
|
7059
|
+
score: link.confidence,
|
|
7060
|
+
created_at: toISOString(link.created_at),
|
|
7061
|
+
metadata: null
|
|
7062
|
+
};
|
|
7063
|
+
}
|
|
7064
|
+
function urlLinkToShard(link) {
|
|
7065
|
+
const metadata = typeof link.metadata_json === "string" ? JSON.parse(link.metadata_json) : link.metadata_json ?? null;
|
|
7066
|
+
return {
|
|
7067
|
+
id: link.id,
|
|
7068
|
+
from_note_id: link.source_note_id,
|
|
7069
|
+
to_note_id: null,
|
|
7070
|
+
to_url: link.to_url,
|
|
6635
7071
|
kind: link.link_type,
|
|
6636
7072
|
score: link.confidence,
|
|
6637
|
-
created_at: toISOString(link.created_at)
|
|
7073
|
+
created_at: toISOString(link.created_at),
|
|
7074
|
+
metadata
|
|
6638
7075
|
};
|
|
6639
7076
|
}
|
|
6640
7077
|
function linkFromShard(shard) {
|
|
@@ -6642,9 +7079,11 @@ function linkFromShard(shard) {
|
|
|
6642
7079
|
id: shard.id,
|
|
6643
7080
|
source_note_id: shard.from_note_id,
|
|
6644
7081
|
target_note_id: shard.to_note_id,
|
|
7082
|
+
to_url: shard.to_url,
|
|
6645
7083
|
link_type: shard.kind,
|
|
6646
7084
|
confidence: shard.score,
|
|
6647
|
-
created_at: shard.created_at
|
|
7085
|
+
created_at: shard.created_at,
|
|
7086
|
+
metadata: shard.metadata
|
|
6648
7087
|
};
|
|
6649
7088
|
}
|
|
6650
7089
|
function collectionToShard(collection, noteCount) {
|
|
@@ -6672,14 +7111,31 @@ function tagsToShard(allTags) {
|
|
|
6672
7111
|
created_at: toISOString(t.created_at)
|
|
6673
7112
|
}));
|
|
6674
7113
|
}
|
|
6675
|
-
function
|
|
6676
|
-
return
|
|
7114
|
+
function templateToShard(template) {
|
|
7115
|
+
return {
|
|
7116
|
+
id: template.id,
|
|
7117
|
+
name: template.name,
|
|
7118
|
+
description: template.description,
|
|
7119
|
+
content: template.content,
|
|
7120
|
+
format: template.format,
|
|
7121
|
+
default_tags: Array.isArray(template.default_tags) ? template.default_tags : JSON.parse(template.default_tags),
|
|
7122
|
+
collection_id: template.collection_id,
|
|
7123
|
+
created_at: toISOString(template.created_at),
|
|
7124
|
+
updated_at: toISOString(template.updated_at)
|
|
7125
|
+
};
|
|
6677
7126
|
}
|
|
6678
7127
|
function embeddingSetToShard(set) {
|
|
7128
|
+
const name = set.name ?? set.model_name;
|
|
6679
7129
|
return {
|
|
6680
7130
|
id: set.id,
|
|
6681
|
-
name
|
|
7131
|
+
name,
|
|
7132
|
+
slug: set.slug ?? slugifyEmbeddingSet(name),
|
|
7133
|
+
description: set.description ?? null,
|
|
6682
7134
|
purpose: set.purpose ?? null,
|
|
7135
|
+
document_count: set.document_count ?? 0,
|
|
7136
|
+
embedding_count: set.embedding_count ?? 0,
|
|
7137
|
+
is_system: set.is_system ?? false,
|
|
7138
|
+
keywords: jsonStringArray(set.keywords_json),
|
|
6683
7139
|
model: set.model_name,
|
|
6684
7140
|
dimension: set.dimensions,
|
|
6685
7141
|
kind: set.kind ?? "physical",
|
|
@@ -6694,11 +7150,17 @@ function embeddingSetToShard(set) {
|
|
|
6694
7150
|
updated_at: set.updated_at ? toISOString(set.updated_at) : void 0
|
|
6695
7151
|
};
|
|
6696
7152
|
}
|
|
6697
|
-
function embeddingSetFromShard(shard) {
|
|
7153
|
+
function embeddingSetFromShard(shard, fallbackCreatedAt) {
|
|
6698
7154
|
return {
|
|
6699
7155
|
id: shard.id,
|
|
6700
7156
|
name: shard.name ?? shard.model,
|
|
7157
|
+
slug: shard.slug ?? null,
|
|
7158
|
+
description: shard.description ?? null,
|
|
6701
7159
|
purpose: shard.purpose ?? null,
|
|
7160
|
+
document_count: shard.document_count ?? null,
|
|
7161
|
+
embedding_count: shard.embedding_count ?? null,
|
|
7162
|
+
is_system: shard.is_system ?? false,
|
|
7163
|
+
keywords_json: jsonString(shard.keywords ?? []),
|
|
6702
7164
|
model_name: shard.model,
|
|
6703
7165
|
dimensions: shard.dimension,
|
|
6704
7166
|
kind: shard.kind ?? "physical",
|
|
@@ -6709,7 +7171,7 @@ function embeddingSetFromShard(shard) {
|
|
|
6709
7171
|
compatibility_json: jsonString(shard.compatibility),
|
|
6710
7172
|
materialization_json: jsonString(shard.materialization),
|
|
6711
7173
|
freshness_json: jsonString(shard.freshness),
|
|
6712
|
-
created_at: shard.created_at,
|
|
7174
|
+
created_at: shard.created_at ?? fallbackCreatedAt,
|
|
6713
7175
|
updated_at: shard.updated_at ?? null
|
|
6714
7176
|
};
|
|
6715
7177
|
}
|
|
@@ -6717,15 +7179,32 @@ function embeddingSetMemberToShard(member) {
|
|
|
6717
7179
|
return {
|
|
6718
7180
|
embedding_set_id: member.embedding_set_id,
|
|
6719
7181
|
note_id: member.note_id,
|
|
6720
|
-
|
|
7182
|
+
membership_type: member.membership_type ?? "materialized",
|
|
7183
|
+
added_at: toISOString(member.added_at ?? /* @__PURE__ */ new Date()),
|
|
7184
|
+
added_by: member.added_by ?? null
|
|
7185
|
+
};
|
|
7186
|
+
}
|
|
7187
|
+
function embeddingConfigToShard(config) {
|
|
7188
|
+
return {
|
|
7189
|
+
id: config.id,
|
|
7190
|
+
name: config.name,
|
|
7191
|
+
description: config.description ?? null,
|
|
7192
|
+
model: config.model,
|
|
7193
|
+
dimension: config.dimension,
|
|
7194
|
+
chunk_size: config.chunk_size,
|
|
7195
|
+
chunk_overlap: config.chunk_overlap,
|
|
7196
|
+
is_default: config.is_default
|
|
6721
7197
|
};
|
|
6722
7198
|
}
|
|
6723
7199
|
function embeddingToShard(emb) {
|
|
6724
7200
|
return {
|
|
6725
7201
|
id: emb.id,
|
|
6726
7202
|
note_id: emb.note_id,
|
|
6727
|
-
|
|
7203
|
+
chunk_index: emb.chunk_index ?? 0,
|
|
7204
|
+
text: emb.text ?? "",
|
|
6728
7205
|
vector: typeof emb.vector === "string" ? parseVector2(emb.vector) : emb.vector,
|
|
7206
|
+
model: emb.model ?? emb.model_name ?? "unknown",
|
|
7207
|
+
embedding_set_id: emb.embedding_set_id,
|
|
6729
7208
|
created_at: toISOString(emb.created_at)
|
|
6730
7209
|
};
|
|
6731
7210
|
}
|
|
@@ -6733,9 +7212,12 @@ function embeddingFromShard(shard) {
|
|
|
6733
7212
|
return {
|
|
6734
7213
|
id: shard.id,
|
|
6735
7214
|
note_id: shard.note_id,
|
|
6736
|
-
embedding_set_id: shard.embedding_set_id,
|
|
7215
|
+
embedding_set_id: shard.embedding_set_id ?? null,
|
|
7216
|
+
chunk_index: shard.chunk_index,
|
|
7217
|
+
text: shard.text,
|
|
6737
7218
|
vector: `[${shard.vector.join(",")}]`,
|
|
6738
|
-
|
|
7219
|
+
model: shard.model,
|
|
7220
|
+
created_at: shard.created_at ?? null
|
|
6739
7221
|
};
|
|
6740
7222
|
}
|
|
6741
7223
|
function skosSchemeToShard(scheme) {
|
|
@@ -6807,6 +7289,15 @@ function parseJsonObjectField(value) {
|
|
|
6807
7289
|
const parsed = JSON.parse(value);
|
|
6808
7290
|
return parsed && typeof parsed === "object" && !Array.isArray(parsed) ? parsed : null;
|
|
6809
7291
|
}
|
|
7292
|
+
function jsonStringArray(value) {
|
|
7293
|
+
if (value == null) return [];
|
|
7294
|
+
if (Array.isArray(value)) return value.map(String);
|
|
7295
|
+
if (typeof value === "string") {
|
|
7296
|
+
const parsed = JSON.parse(value);
|
|
7297
|
+
return Array.isArray(parsed) ? parsed.map(String) : [];
|
|
7298
|
+
}
|
|
7299
|
+
return [];
|
|
7300
|
+
}
|
|
6810
7301
|
function jsonObject2(value) {
|
|
6811
7302
|
if (value == null) return null;
|
|
6812
7303
|
if (typeof value === "string") return JSON.parse(value);
|
|
@@ -6815,6 +7306,34 @@ function jsonObject2(value) {
|
|
|
6815
7306
|
function jsonString(value) {
|
|
6816
7307
|
return value == null ? null : JSON.stringify(value);
|
|
6817
7308
|
}
|
|
7309
|
+
function slugifyEmbeddingSet(value) {
|
|
7310
|
+
const slug = value.trim().toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "");
|
|
7311
|
+
return slug || "embedding-set";
|
|
7312
|
+
}
|
|
7313
|
+
|
|
7314
|
+
// src/shard/blob-sidecar.ts
|
|
7315
|
+
var SIDECAR_PREFIX = "blobs/";
|
|
7316
|
+
function blobChecksumToHex(checksum) {
|
|
7317
|
+
const sep = checksum.indexOf(":");
|
|
7318
|
+
return sep >= 0 ? checksum.slice(sep + 1) : checksum;
|
|
7319
|
+
}
|
|
7320
|
+
function sidecarEntryName(checksum) {
|
|
7321
|
+
return SIDECAR_PREFIX + blobChecksumToHex(checksum);
|
|
7322
|
+
}
|
|
7323
|
+
function isSidecarEntry(name) {
|
|
7324
|
+
if (!name.startsWith(SIDECAR_PREFIX)) return false;
|
|
7325
|
+
const rest = name.slice(SIDECAR_PREFIX.length);
|
|
7326
|
+
return rest.length > 0 && !rest.includes("/");
|
|
7327
|
+
}
|
|
7328
|
+
function collectSidecarBlobs(files) {
|
|
7329
|
+
const blobs = /* @__PURE__ */ new Map();
|
|
7330
|
+
for (const [name, bytes] of files) {
|
|
7331
|
+
if (isSidecarEntry(name)) {
|
|
7332
|
+
blobs.set(name.slice(SIDECAR_PREFIX.length), bytes);
|
|
7333
|
+
}
|
|
7334
|
+
}
|
|
7335
|
+
return blobs;
|
|
7336
|
+
}
|
|
6818
7337
|
|
|
6819
7338
|
// src/shard/shard-export.ts
|
|
6820
7339
|
var encoder = new TextEncoder();
|
|
@@ -6836,7 +7355,8 @@ async function exportShard(db, options) {
|
|
|
6836
7355
|
noteQuery = `SELECT n.id, n.title, n.format, n.source, n.is_starred, n.is_archived,
|
|
6837
7356
|
n.created_at, n.updated_at, n.deleted_at,
|
|
6838
7357
|
o.content as original_content,
|
|
6839
|
-
c.content as revised_content
|
|
7358
|
+
c.content as revised_content,
|
|
7359
|
+
$1::text as collection_id
|
|
6840
7360
|
FROM note n
|
|
6841
7361
|
LEFT JOIN note_original o ON o.note_id = n.id
|
|
6842
7362
|
LEFT JOIN note_revised_current c ON c.note_id = n.id
|
|
@@ -6848,7 +7368,14 @@ async function exportShard(db, options) {
|
|
|
6848
7368
|
noteQuery = `SELECT n.id, n.title, n.format, n.source, n.is_starred, n.is_archived,
|
|
6849
7369
|
n.created_at, n.updated_at, n.deleted_at,
|
|
6850
7370
|
o.content as original_content,
|
|
6851
|
-
c.content as revised_content
|
|
7371
|
+
c.content as revised_content,
|
|
7372
|
+
(
|
|
7373
|
+
SELECT cn.collection_id
|
|
7374
|
+
FROM collection_note cn
|
|
7375
|
+
WHERE cn.note_id = n.id
|
|
7376
|
+
ORDER BY cn.position, cn.added_at
|
|
7377
|
+
LIMIT 1
|
|
7378
|
+
) as collection_id
|
|
6852
7379
|
FROM note n
|
|
6853
7380
|
LEFT JOIN note_original o ON o.note_id = n.id
|
|
6854
7381
|
LEFT JOIN note_revised_current c ON c.note_id = n.id
|
|
@@ -6860,7 +7387,14 @@ async function exportShard(db, options) {
|
|
|
6860
7387
|
noteQuery = `SELECT n.id, n.title, n.format, n.source, n.is_starred, n.is_archived,
|
|
6861
7388
|
n.created_at, n.updated_at, n.deleted_at,
|
|
6862
7389
|
o.content as original_content,
|
|
6863
|
-
c.content as revised_content
|
|
7390
|
+
c.content as revised_content,
|
|
7391
|
+
(
|
|
7392
|
+
SELECT cn.collection_id
|
|
7393
|
+
FROM collection_note cn
|
|
7394
|
+
WHERE cn.note_id = n.id
|
|
7395
|
+
ORDER BY cn.position, cn.added_at
|
|
7396
|
+
LIMIT 1
|
|
7397
|
+
) as collection_id
|
|
6864
7398
|
FROM note n
|
|
6865
7399
|
LEFT JOIN note_original o ON o.note_id = n.id
|
|
6866
7400
|
LEFT JOIN note_revised_current c ON c.note_id = n.id
|
|
@@ -6892,26 +7426,28 @@ async function exportShard(db, options) {
|
|
|
6892
7426
|
WHERE a.deleted_at IS NULL
|
|
6893
7427
|
ORDER BY a.note_id, a.position, a.created_at`
|
|
6894
7428
|
);
|
|
6895
|
-
const
|
|
7429
|
+
const attachmentsByNote = /* @__PURE__ */ new Map();
|
|
6896
7430
|
for (const row of attachmentRows.rows) {
|
|
6897
7431
|
const source = {
|
|
6898
|
-
extracted_text: row.extracted_text
|
|
7432
|
+
extracted_text: row.extracted_text,
|
|
6899
7433
|
attachment: {
|
|
7434
|
+
// `path` is the display filename per the binary-attachment projection
|
|
7435
|
+
// contract — never the physical storage key (`storage_path`).
|
|
6900
7436
|
id: row.id,
|
|
6901
|
-
path: row.
|
|
7437
|
+
path: row.filename,
|
|
6902
7438
|
mime: row.mime_type,
|
|
6903
7439
|
checksum: row.content_hash,
|
|
6904
7440
|
bytes: Number(row.size_bytes)
|
|
6905
7441
|
}
|
|
6906
7442
|
};
|
|
6907
|
-
const sources =
|
|
7443
|
+
const sources = attachmentsByNote.get(row.note_id) ?? [];
|
|
6908
7444
|
sources.push(source);
|
|
6909
|
-
|
|
7445
|
+
attachmentsByNote.set(row.note_id, sources);
|
|
6910
7446
|
}
|
|
6911
7447
|
const notes = noteRows.rows.map((row) => ({
|
|
6912
7448
|
...row,
|
|
6913
7449
|
tags: tagsByNote.get(row.id) ?? [],
|
|
6914
|
-
|
|
7450
|
+
attachments: attachmentsByNote.get(row.id)
|
|
6915
7451
|
}));
|
|
6916
7452
|
const exportedNoteIds = new Set(notes.map((n) => n.id));
|
|
6917
7453
|
const shardNotes = notes.map((n) => noteToShard(n));
|
|
@@ -6922,7 +7458,7 @@ async function exportShard(db, options) {
|
|
|
6922
7458
|
for (let offset = 0; offset < shardNotes.length; offset += clusterSize) {
|
|
6923
7459
|
const slice = shardNotes.slice(offset, offset + clusterSize);
|
|
6924
7460
|
const href = `notes/${String(offset).padStart(6, "0")}.jsonl`;
|
|
6925
|
-
clusters.push({ href, offset
|
|
7461
|
+
clusters.push({ href, offset });
|
|
6926
7462
|
files.set(href, encoder.encode(slice.map((n) => JSON.stringify(n)).join("\n")));
|
|
6927
7463
|
}
|
|
6928
7464
|
layout = { clusters: { notes: clusters } };
|
|
@@ -6963,14 +7499,27 @@ async function exportShard(db, options) {
|
|
|
6963
7499
|
files.set("tags.json", encoder.encode(JSON.stringify(shardTags)));
|
|
6964
7500
|
components.push("tags");
|
|
6965
7501
|
counts.tags = shardTags.length;
|
|
7502
|
+
const templateRows = await db.query(`SELECT * FROM template ORDER BY created_at, id`);
|
|
7503
|
+
if (templateRows.rows.length > 0) {
|
|
7504
|
+
const shardTemplates = templateRows.rows.map((template) => templateToShard(template));
|
|
7505
|
+
files.set("templates.json", encoder.encode(JSON.stringify(shardTemplates)));
|
|
7506
|
+
components.push("templates");
|
|
7507
|
+
counts.templates = shardTemplates.length;
|
|
7508
|
+
}
|
|
6966
7509
|
const linkRows = await db.query(
|
|
6967
7510
|
`SELECT * FROM link WHERE deleted_at IS NULL ORDER BY created_at`
|
|
6968
7511
|
);
|
|
6969
7512
|
const filteredLinks = options?.tag || options?.collectionId ? linkRows.rows.filter((l) => exportedNoteIds.has(l.source_note_id) && exportedNoteIds.has(l.target_note_id)) : linkRows.rows;
|
|
6970
|
-
const
|
|
7513
|
+
const urlLinkRows = await db.query(`SELECT * FROM link_url_target WHERE deleted_at IS NULL ORDER BY created_at`);
|
|
7514
|
+
const filteredUrlLinks = options?.tag || options?.collectionId ? urlLinkRows.rows.filter((l) => exportedNoteIds.has(l.source_note_id)) : urlLinkRows.rows;
|
|
7515
|
+
const shardLinks = [
|
|
7516
|
+
...filteredLinks.map((l) => linkToShard(l)),
|
|
7517
|
+
...filteredUrlLinks.map((l) => urlLinkToShard(l))
|
|
7518
|
+
];
|
|
7519
|
+
const linksJsonl = shardLinks.map((l) => JSON.stringify(l)).join("\n");
|
|
6971
7520
|
files.set("links.jsonl", encoder.encode(linksJsonl));
|
|
6972
7521
|
components.push("links");
|
|
6973
|
-
counts.links =
|
|
7522
|
+
counts.links = shardLinks.length;
|
|
6974
7523
|
const allNoteSkosRows = await db.query(`SELECT * FROM note_skos_tag ORDER BY created_at`);
|
|
6975
7524
|
const filteredNoteSkosRows = isFiltered ? allNoteSkosRows.rows.filter((row) => exportedNoteIds.has(row.note_id)) : allNoteSkosRows.rows;
|
|
6976
7525
|
const referencedConceptIds = new Set(filteredNoteSkosRows.map((row) => row.concept_id));
|
|
@@ -7011,9 +7560,27 @@ async function exportShard(db, options) {
|
|
|
7011
7560
|
const setScoped = embeddingSetIds.length > 0;
|
|
7012
7561
|
const includeMaterializedSelectors = options.includeMaterializedSelectors === true;
|
|
7013
7562
|
const embSetRows = await db.query(
|
|
7014
|
-
`SELECT
|
|
7015
|
-
|
|
7016
|
-
|
|
7563
|
+
`SELECT
|
|
7564
|
+
es.id, es.name, es.slug, es.description, es.purpose,
|
|
7565
|
+
COALESCE(es.document_count, member_counts.document_count, 0)::int AS document_count,
|
|
7566
|
+
COALESCE(es.embedding_count, embedding_counts.embedding_count, 0)::int AS embedding_count,
|
|
7567
|
+
es.is_system, es.keywords_json,
|
|
7568
|
+
es.model_name, es.dimensions, es.kind, es.mode, es.truncate_dimension,
|
|
7569
|
+
es.criteria_json, es.source_json, es.compatibility_json, es.materialization_json,
|
|
7570
|
+
es.freshness_json, es.created_at, es.updated_at
|
|
7571
|
+
FROM embedding_set es
|
|
7572
|
+
LEFT JOIN (
|
|
7573
|
+
SELECT embedding_set_id, COUNT(*)::int AS document_count
|
|
7574
|
+
FROM embedding_set_member
|
|
7575
|
+
GROUP BY embedding_set_id
|
|
7576
|
+
) member_counts ON member_counts.embedding_set_id = es.id
|
|
7577
|
+
LEFT JOIN (
|
|
7578
|
+
SELECT embedding_set_id, COUNT(*)::int AS embedding_count
|
|
7579
|
+
FROM embedding
|
|
7580
|
+
GROUP BY embedding_set_id
|
|
7581
|
+
) embedding_counts ON embedding_counts.embedding_set_id = es.id
|
|
7582
|
+
${setScoped ? "WHERE es.id = ANY($1)" : ""}
|
|
7583
|
+
ORDER BY es.created_at`,
|
|
7017
7584
|
setScoped ? [embeddingSetIds] : []
|
|
7018
7585
|
);
|
|
7019
7586
|
const exportedSetIds = new Set(embSetRows.rows.map((row) => row.id));
|
|
@@ -7028,6 +7595,17 @@ async function exportShard(db, options) {
|
|
|
7028
7595
|
files.set("embedding_sets.json", encoder.encode(JSON.stringify(shardEmbSets)));
|
|
7029
7596
|
components.push("embedding_sets");
|
|
7030
7597
|
counts.embedding_sets = shardEmbSets.length;
|
|
7598
|
+
const embeddingConfigRows = await db.query(
|
|
7599
|
+
`SELECT id, name, description, model, dimension, chunk_size, chunk_overlap, is_default
|
|
7600
|
+
FROM embedding_config
|
|
7601
|
+
ORDER BY name, id`
|
|
7602
|
+
);
|
|
7603
|
+
if (embeddingConfigRows.rows.length > 0) {
|
|
7604
|
+
const shardEmbeddingConfigs = embeddingConfigRows.rows.map((row) => embeddingConfigToShard(row));
|
|
7605
|
+
files.set("embedding_configs.json", encoder.encode(JSON.stringify(shardEmbeddingConfigs)));
|
|
7606
|
+
components.push("embedding_configs");
|
|
7607
|
+
counts.embedding_configs = shardEmbeddingConfigs.length;
|
|
7608
|
+
}
|
|
7031
7609
|
const embMemberRows = await db.query(
|
|
7032
7610
|
`SELECT * FROM embedding_set_member
|
|
7033
7611
|
${setScoped ? "WHERE embedding_set_id = ANY($1)" : ""}`,
|
|
@@ -7041,12 +7619,24 @@ async function exportShard(db, options) {
|
|
|
7041
7619
|
components.push("embedding_set_members");
|
|
7042
7620
|
counts.embedding_set_members = scopedEmbMemberRows.length;
|
|
7043
7621
|
const embRows = await db.query(
|
|
7044
|
-
`SELECT
|
|
7045
|
-
|
|
7046
|
-
|
|
7622
|
+
`SELECT
|
|
7623
|
+
e.id, e.note_id, e.embedding_set_id, e.chunk_index,
|
|
7624
|
+
COALESCE(NULLIF(e.text, ''), nrc.content, no.content, '') AS text,
|
|
7625
|
+
e.vector,
|
|
7626
|
+
COALESCE(e.model, es.model_name) AS model,
|
|
7627
|
+
es.model_name,
|
|
7628
|
+
e.created_at
|
|
7629
|
+
FROM embedding e
|
|
7630
|
+
JOIN embedding_set es ON es.id = e.embedding_set_id
|
|
7631
|
+
LEFT JOIN note_revised_current nrc ON nrc.note_id = e.note_id
|
|
7632
|
+
LEFT JOIN note_original no ON no.note_id = e.note_id
|
|
7633
|
+
${setScoped ? "WHERE e.embedding_set_id = ANY($1)" : ""}
|
|
7634
|
+
ORDER BY e.created_at`,
|
|
7047
7635
|
setScoped ? [embeddingSetIds] : []
|
|
7048
7636
|
);
|
|
7049
|
-
const memberEmbeddingIds = new Set(
|
|
7637
|
+
const memberEmbeddingIds = new Set(
|
|
7638
|
+
scopedEmbMemberRows.map((member) => member.embedding_id).filter((id) => Boolean(id))
|
|
7639
|
+
);
|
|
7050
7640
|
const scopedEmbRows = embRows.rows.filter(
|
|
7051
7641
|
(embedding) => exportedSetIds.has(embedding.embedding_set_id) && exportedNoteIds.has(embedding.note_id) && (memberEmbeddingIds.size === 0 || memberEmbeddingIds.has(embedding.id))
|
|
7052
7642
|
);
|
|
@@ -7162,14 +7752,37 @@ async function exportShard(db, options) {
|
|
|
7162
7752
|
counts,
|
|
7163
7753
|
checksums,
|
|
7164
7754
|
min_reader_version: "1.0.0",
|
|
7755
|
+
migrated_from: null,
|
|
7756
|
+
migration_history: [],
|
|
7165
7757
|
...layout ? { layout } : {}
|
|
7166
7758
|
};
|
|
7167
7759
|
files.set("manifest.json", encoder.encode(JSON.stringify(manifest, null, 2)));
|
|
7760
|
+
if (options?.includeBlobs && options.blobStore) {
|
|
7761
|
+
const packed = /* @__PURE__ */ new Set();
|
|
7762
|
+
for (const row of attachmentRows.rows) {
|
|
7763
|
+
const checksum = row.content_hash;
|
|
7764
|
+
if (packed.has(checksum)) continue;
|
|
7765
|
+
packed.add(checksum);
|
|
7766
|
+
const bytes = await options.blobStore.read(checksum);
|
|
7767
|
+
if (bytes) files.set(sidecarEntryName(checksum), bytes);
|
|
7768
|
+
}
|
|
7769
|
+
}
|
|
7168
7770
|
return packTarGz(files);
|
|
7169
7771
|
}
|
|
7170
7772
|
|
|
7171
|
-
// src/shard/
|
|
7773
|
+
// src/shard/parse.ts
|
|
7172
7774
|
var decoder = new TextDecoder();
|
|
7775
|
+
function parseJsonlBytes(data) {
|
|
7776
|
+
if (!data || data.byteLength === 0) return [];
|
|
7777
|
+
return decoder.decode(data).split("\n").filter((line) => line.trim()).map((line) => JSON.parse(line));
|
|
7778
|
+
}
|
|
7779
|
+
function parseJsonArrayBytes(data) {
|
|
7780
|
+
if (!data || data.byteLength === 0) return [];
|
|
7781
|
+
return JSON.parse(decoder.decode(data));
|
|
7782
|
+
}
|
|
7783
|
+
|
|
7784
|
+
// src/shard/shard-import.ts
|
|
7785
|
+
var decoder2 = new TextDecoder();
|
|
7173
7786
|
var DEFAULT_BATCH_SIZE = 250;
|
|
7174
7787
|
async function yieldToEventLoop2() {
|
|
7175
7788
|
const scheduler = globalThis.scheduler;
|
|
@@ -7191,14 +7804,17 @@ async function importShard(db, data, options) {
|
|
|
7191
7804
|
const report = options?.onProgress;
|
|
7192
7805
|
const warnings = [];
|
|
7193
7806
|
const errors = [];
|
|
7194
|
-
let
|
|
7195
|
-
let
|
|
7807
|
+
let importedAttachmentReferenceCount = 0;
|
|
7808
|
+
let notesWithImportedAttachmentReferences = 0;
|
|
7809
|
+
const blobsToHydrate = /* @__PURE__ */ new Map();
|
|
7196
7810
|
const counts = {
|
|
7197
7811
|
notes: 0,
|
|
7198
7812
|
collections: 0,
|
|
7813
|
+
templates: 0,
|
|
7199
7814
|
tags: 0,
|
|
7200
7815
|
links: 0,
|
|
7201
7816
|
embedding_sets: 0,
|
|
7817
|
+
embedding_configs: 0,
|
|
7202
7818
|
embedding_set_members: 0,
|
|
7203
7819
|
embeddings: 0,
|
|
7204
7820
|
skos_schemes: 0,
|
|
@@ -7242,7 +7858,10 @@ async function importShard(db, data, options) {
|
|
|
7242
7858
|
}
|
|
7243
7859
|
let manifest;
|
|
7244
7860
|
try {
|
|
7245
|
-
manifest = JSON.parse(
|
|
7861
|
+
manifest = JSON.parse(decoder2.decode(manifestData));
|
|
7862
|
+
if (!manifest || typeof manifest !== "object" || Array.isArray(manifest)) {
|
|
7863
|
+
throw new Error("manifest must be a JSON object");
|
|
7864
|
+
}
|
|
7246
7865
|
} catch {
|
|
7247
7866
|
return {
|
|
7248
7867
|
success: false,
|
|
@@ -7253,7 +7872,7 @@ async function importShard(db, data, options) {
|
|
|
7253
7872
|
duration_ms: performance.now() - start
|
|
7254
7873
|
};
|
|
7255
7874
|
}
|
|
7256
|
-
if (manifest.min_reader_version && manifest.min_reader_version >
|
|
7875
|
+
if (manifest.min_reader_version && compareShardVersions(manifest.min_reader_version, CURRENT_SHARD_VERSION) > 0) {
|
|
7257
7876
|
return {
|
|
7258
7877
|
success: false,
|
|
7259
7878
|
counts,
|
|
@@ -7266,6 +7885,16 @@ async function importShard(db, data, options) {
|
|
|
7266
7885
|
};
|
|
7267
7886
|
}
|
|
7268
7887
|
report?.({ phase: "validate", done: 0, total: 1 });
|
|
7888
|
+
if (!manifest.checksums || typeof manifest.checksums !== "object") {
|
|
7889
|
+
return {
|
|
7890
|
+
success: false,
|
|
7891
|
+
counts,
|
|
7892
|
+
skipped,
|
|
7893
|
+
warnings,
|
|
7894
|
+
errors: ["Invalid manifest.json: checksums must be an object"],
|
|
7895
|
+
duration_ms: performance.now() - start
|
|
7896
|
+
};
|
|
7897
|
+
}
|
|
7269
7898
|
const checksumResult = await validateChecksums(manifest.checksums, files);
|
|
7270
7899
|
if (!checksumResult.valid) {
|
|
7271
7900
|
return {
|
|
@@ -7278,25 +7907,61 @@ async function importShard(db, data, options) {
|
|
|
7278
7907
|
};
|
|
7279
7908
|
}
|
|
7280
7909
|
report?.({ phase: "validate", done: 1, total: 1 });
|
|
7910
|
+
const sidecarBlobs = options?.blobStore ? collectSidecarBlobs(files) : null;
|
|
7281
7911
|
const noteClusters = manifest.layout?.clusters?.notes;
|
|
7282
|
-
const
|
|
7283
|
-
|
|
7284
|
-
|
|
7285
|
-
|
|
7286
|
-
|
|
7287
|
-
|
|
7288
|
-
|
|
7289
|
-
|
|
7290
|
-
|
|
7291
|
-
|
|
7292
|
-
|
|
7293
|
-
|
|
7294
|
-
|
|
7295
|
-
|
|
7296
|
-
|
|
7297
|
-
|
|
7298
|
-
|
|
7299
|
-
|
|
7912
|
+
const parseComponent = (name, parse) => {
|
|
7913
|
+
try {
|
|
7914
|
+
return parse();
|
|
7915
|
+
} catch (err) {
|
|
7916
|
+
throw new Error(`${name}: ${err instanceof Error ? err.message : String(err)}`);
|
|
7917
|
+
}
|
|
7918
|
+
};
|
|
7919
|
+
let parsedNotes;
|
|
7920
|
+
let parsedCollections;
|
|
7921
|
+
let parsedTemplates;
|
|
7922
|
+
let parsedLinks;
|
|
7923
|
+
let parsedEmbSets;
|
|
7924
|
+
let parsedEmbConfigs;
|
|
7925
|
+
let parsedEmbMembers;
|
|
7926
|
+
let parsedEmbeddings;
|
|
7927
|
+
let parsedSkosSchemes;
|
|
7928
|
+
let parsedSkosConcepts;
|
|
7929
|
+
let parsedSkosRelations;
|
|
7930
|
+
let parsedNoteSkosTags;
|
|
7931
|
+
let parsedProvenanceEdges;
|
|
7932
|
+
let parsedGraphSources;
|
|
7933
|
+
let parsedGraphEdges;
|
|
7934
|
+
let parsedCommunitySets;
|
|
7935
|
+
let parsedCommunityAssignments;
|
|
7936
|
+
try {
|
|
7937
|
+
parsedNotes = parseComponent("notes", () => noteClusters && noteClusters.length > 0 ? [...noteClusters].sort((a, b) => a.offset - b.offset).flatMap((ref) => parseJsonlBytes(files.get(ref.href))) : parseJsonlBytes(files.get("notes.jsonl")));
|
|
7938
|
+
parsedCollections = parseComponent("collections.json", () => parseJsonArrayBytes(files.get("collections.json")));
|
|
7939
|
+
parseComponent("tags.json", () => parseJsonArrayBytes(files.get("tags.json")));
|
|
7940
|
+
parsedTemplates = parseComponent("templates.json", () => parseJsonArrayBytes(files.get("templates.json")));
|
|
7941
|
+
parsedLinks = parseComponent("links.jsonl", () => parseJsonlBytes(files.get("links.jsonl")));
|
|
7942
|
+
parsedEmbSets = parseComponent("embedding_sets.json", () => parseJsonArrayBytes(files.get("embedding_sets.json")));
|
|
7943
|
+
parsedEmbConfigs = parseComponent("embedding_configs.json", () => parseJsonArrayBytes(files.get("embedding_configs.json")));
|
|
7944
|
+
parsedEmbMembers = parseComponent("embedding_set_members.jsonl", () => parseJsonlBytes(files.get("embedding_set_members.jsonl")));
|
|
7945
|
+
parsedEmbeddings = parseComponent("embeddings.jsonl", () => parseJsonlBytes(files.get("embeddings.jsonl")));
|
|
7946
|
+
parsedSkosSchemes = parseComponent("skos_schemes.json", () => parseJsonArrayBytes(files.get("skos_schemes.json")));
|
|
7947
|
+
parsedSkosConcepts = parseComponent("skos_concepts.json", () => parseJsonArrayBytes(files.get("skos_concepts.json")));
|
|
7948
|
+
parsedSkosRelations = parseComponent("skos_relations.jsonl", () => parseJsonlBytes(files.get("skos_relations.jsonl")));
|
|
7949
|
+
parsedNoteSkosTags = parseComponent("note_skos_tags.jsonl", () => parseJsonlBytes(files.get("note_skos_tags.jsonl")));
|
|
7950
|
+
parsedProvenanceEdges = parseComponent("provenance_edges.jsonl", () => parseJsonlBytes(files.get("provenance_edges.jsonl")));
|
|
7951
|
+
parsedGraphSources = parseComponent("graph_sources.json", () => parseJsonArrayBytes(files.get("graph_sources.json")));
|
|
7952
|
+
parsedGraphEdges = parseComponent("graph_edges.jsonl", () => parseJsonlBytes(files.get("graph_edges.jsonl")));
|
|
7953
|
+
parsedCommunitySets = parseComponent("communities.json", () => parseJsonArrayBytes(files.get("communities.json")));
|
|
7954
|
+
parsedCommunityAssignments = parseComponent("community_assignments.jsonl", () => parseJsonlBytes(files.get("community_assignments.jsonl")));
|
|
7955
|
+
} catch (err) {
|
|
7956
|
+
return {
|
|
7957
|
+
success: false,
|
|
7958
|
+
counts,
|
|
7959
|
+
skipped,
|
|
7960
|
+
warnings,
|
|
7961
|
+
errors: [`Failed to parse shard component: ${err instanceof Error ? err.message : String(err)}`],
|
|
7962
|
+
duration_ms: performance.now() - start
|
|
7963
|
+
};
|
|
7964
|
+
}
|
|
7300
7965
|
const knownFiles = /* @__PURE__ */ new Set([
|
|
7301
7966
|
"manifest.json",
|
|
7302
7967
|
"notes.jsonl",
|
|
@@ -7305,7 +7970,6 @@ async function importShard(db, data, options) {
|
|
|
7305
7970
|
"links.jsonl",
|
|
7306
7971
|
"embedding_sets.json",
|
|
7307
7972
|
"embedding_set_members.jsonl",
|
|
7308
|
-
"embedding_configs.json",
|
|
7309
7973
|
"embeddings.jsonl",
|
|
7310
7974
|
"templates.json",
|
|
7311
7975
|
"skos_schemes.json",
|
|
@@ -7323,15 +7987,36 @@ async function importShard(db, data, options) {
|
|
|
7323
7987
|
warnings.push(`Unknown component skipped: ${filename}`);
|
|
7324
7988
|
}
|
|
7325
7989
|
}
|
|
7326
|
-
if (files.has("templates.json")) {
|
|
7327
|
-
warnings.push("templates.json skipped (not supported in browser)");
|
|
7328
|
-
}
|
|
7329
7990
|
const conflictClause = strategy === "skip" ? "ON CONFLICT DO NOTHING" : "";
|
|
7330
7991
|
try {
|
|
7331
7992
|
await db.transaction(async (tx) => {
|
|
7993
|
+
const preexistingEmbeddingMembers = /* @__PURE__ */ new Set();
|
|
7994
|
+
if (strategy === "skip") {
|
|
7995
|
+
for (const member of parsedEmbMembers) {
|
|
7996
|
+
const existing = await tx.query(
|
|
7997
|
+
"SELECT 1 FROM embedding_set_member WHERE embedding_set_id = $1 AND note_id = $2",
|
|
7998
|
+
[member.embedding_set_id, member.note_id]
|
|
7999
|
+
);
|
|
8000
|
+
if (existing.rows.length > 0) {
|
|
8001
|
+
preexistingEmbeddingMembers.add(`${member.embedding_set_id}\0${member.note_id}`);
|
|
8002
|
+
}
|
|
8003
|
+
}
|
|
8004
|
+
}
|
|
8005
|
+
const skipExisting = async (key, sql, params) => {
|
|
8006
|
+
if (strategy !== "skip") return false;
|
|
8007
|
+
const existing = await tx.query(sql, params);
|
|
8008
|
+
if (existing.rows.length === 0) return false;
|
|
8009
|
+
skipped[key] = (skipped[key] ?? 0) + 1;
|
|
8010
|
+
return true;
|
|
8011
|
+
};
|
|
7332
8012
|
report?.({ phase: "collections", done: 0, total: parsedCollections.length });
|
|
7333
8013
|
for (const [index, shardCol] of parsedCollections.entries()) {
|
|
7334
8014
|
const col = collectionFromShard(shardCol);
|
|
8015
|
+
if (await skipExisting("collections", "SELECT 1 FROM collection WHERE id = $1", [col.id])) {
|
|
8016
|
+
report?.({ phase: "collections", done: index + 1, total: parsedCollections.length });
|
|
8017
|
+
await maybeYield2(index + 1, batchSize);
|
|
8018
|
+
continue;
|
|
8019
|
+
}
|
|
7335
8020
|
if (strategy === "replace") {
|
|
7336
8021
|
await tx.query(
|
|
7337
8022
|
`INSERT INTO collection (id, name, description, parent_id, created_at)
|
|
@@ -7350,9 +8035,38 @@ async function importShard(db, data, options) {
|
|
|
7350
8035
|
report?.({ phase: "collections", done: index + 1, total: parsedCollections.length });
|
|
7351
8036
|
await maybeYield2(index + 1, batchSize);
|
|
7352
8037
|
}
|
|
8038
|
+
report?.({ phase: "templates", done: 0, total: parsedTemplates.length });
|
|
8039
|
+
for (const [index, template] of parsedTemplates.entries()) {
|
|
8040
|
+
await tx.query(
|
|
8041
|
+
`INSERT INTO template (
|
|
8042
|
+
id, name, description, content, format, default_tags, collection_id, created_at, updated_at
|
|
8043
|
+
)
|
|
8044
|
+
VALUES ($1, $2, $3, $4, $5, $6::jsonb, $7, $8, $9)
|
|
8045
|
+
${strategy === "replace" ? "ON CONFLICT (id) DO UPDATE SET name = $2, description = $3, content = $4, format = $5, default_tags = $6::jsonb, collection_id = $7, created_at = $8, updated_at = $9" : conflictClause}`,
|
|
8046
|
+
[
|
|
8047
|
+
template.id,
|
|
8048
|
+
template.name,
|
|
8049
|
+
template.description,
|
|
8050
|
+
template.content,
|
|
8051
|
+
template.format,
|
|
8052
|
+
JSON.stringify(template.default_tags),
|
|
8053
|
+
template.collection_id,
|
|
8054
|
+
template.created_at,
|
|
8055
|
+
template.updated_at
|
|
8056
|
+
]
|
|
8057
|
+
);
|
|
8058
|
+
counts.templates++;
|
|
8059
|
+
report?.({ phase: "templates", done: index + 1, total: parsedTemplates.length });
|
|
8060
|
+
await maybeYield2(index + 1, batchSize);
|
|
8061
|
+
}
|
|
7353
8062
|
report?.({ phase: "notes", done: 0, total: parsedNotes.length });
|
|
7354
8063
|
for (const [index, shardNote] of parsedNotes.entries()) {
|
|
7355
8064
|
const note = noteFromShard(shardNote);
|
|
8065
|
+
if (await skipExisting("notes", "SELECT 1 FROM note WHERE id = $1", [note.id])) {
|
|
8066
|
+
report?.({ phase: "notes", done: index + 1, total: parsedNotes.length });
|
|
8067
|
+
await maybeYield2(index + 1, batchSize);
|
|
8068
|
+
continue;
|
|
8069
|
+
}
|
|
7356
8070
|
const contentHash = computeHash(new TextEncoder().encode(note.original_content));
|
|
7357
8071
|
if (strategy === "replace") {
|
|
7358
8072
|
await tx.query(
|
|
@@ -7427,23 +8141,98 @@ async function importShard(db, data, options) {
|
|
|
7427
8141
|
[generateId(), note.id, tag]
|
|
7428
8142
|
);
|
|
7429
8143
|
}
|
|
7430
|
-
if (note.
|
|
7431
|
-
|
|
7432
|
-
|
|
8144
|
+
if (note.collection_id) {
|
|
8145
|
+
await tx.query(
|
|
8146
|
+
`INSERT INTO collection_note (collection_id, note_id)
|
|
8147
|
+
VALUES ($1, $2)
|
|
8148
|
+
ON CONFLICT (collection_id, note_id) DO NOTHING`,
|
|
8149
|
+
[note.collection_id, note.id]
|
|
8150
|
+
);
|
|
8151
|
+
}
|
|
8152
|
+
if (note.attachments?.length) {
|
|
8153
|
+
notesWithImportedAttachmentReferences++;
|
|
8154
|
+
for (let position = 0; position < note.attachments.length; position += 1) {
|
|
8155
|
+
const projection = note.attachments[position];
|
|
8156
|
+
const ref = projection.attachment;
|
|
8157
|
+
const existingBlob = await tx.query(
|
|
8158
|
+
`SELECT id FROM attachment_blob WHERE content_hash = $1`,
|
|
8159
|
+
[ref.checksum]
|
|
8160
|
+
);
|
|
8161
|
+
const blobId = existingBlob.rows[0]?.id ?? generateId();
|
|
8162
|
+
if (!existingBlob.rows.length) {
|
|
8163
|
+
await tx.query(
|
|
8164
|
+
`INSERT INTO attachment_blob (id, content_hash, size_bytes, storage_path)
|
|
8165
|
+
VALUES ($1, $2, $3, $4)
|
|
8166
|
+
ON CONFLICT (content_hash) DO NOTHING`,
|
|
8167
|
+
// storage_path stays NULL: the browser edition addresses blobs
|
|
8168
|
+
// by content_hash via the BlobStore, not a filesystem key, and
|
|
8169
|
+
// `ref.path` is the display filename (never a storage locator).
|
|
8170
|
+
[blobId, ref.checksum, ref.bytes, null]
|
|
8171
|
+
);
|
|
8172
|
+
}
|
|
8173
|
+
const filename = ref.path.split("/").filter(Boolean).pop() ?? ref.path;
|
|
8174
|
+
if (strategy === "replace") {
|
|
8175
|
+
await tx.query(
|
|
8176
|
+
`INSERT INTO attachment (id, note_id, blob_id, filename, mime_type, extracted_text, position)
|
|
8177
|
+
VALUES ($1, $2, $3, $4, $5, $6, $7)
|
|
8178
|
+
ON CONFLICT (id) DO UPDATE SET
|
|
8179
|
+
note_id = $2,
|
|
8180
|
+
blob_id = $3,
|
|
8181
|
+
filename = $4,
|
|
8182
|
+
mime_type = $5,
|
|
8183
|
+
extracted_text = $6,
|
|
8184
|
+
position = $7,
|
|
8185
|
+
deleted_at = NULL`,
|
|
8186
|
+
[ref.id, note.id, blobId, filename, ref.mime, projection.extracted_text, position]
|
|
8187
|
+
);
|
|
8188
|
+
} else {
|
|
8189
|
+
await tx.query(
|
|
8190
|
+
`INSERT INTO attachment (id, note_id, blob_id, filename, mime_type, extracted_text, position)
|
|
8191
|
+
VALUES ($1, $2, $3, $4, $5, $6, $7) ${conflictClause}`,
|
|
8192
|
+
[ref.id, note.id, blobId, filename, ref.mime, projection.extracted_text, position]
|
|
8193
|
+
);
|
|
8194
|
+
}
|
|
8195
|
+
let hydrated = false;
|
|
8196
|
+
if (sidecarBlobs) {
|
|
8197
|
+
if (blobsToHydrate.has(ref.checksum)) {
|
|
8198
|
+
hydrated = true;
|
|
8199
|
+
} else {
|
|
8200
|
+
const bytes = sidecarBlobs.get(blobChecksumToHex(ref.checksum));
|
|
8201
|
+
if (bytes) {
|
|
8202
|
+
if (computeBlobHash(bytes) === ref.checksum) {
|
|
8203
|
+
blobsToHydrate.set(ref.checksum, bytes);
|
|
8204
|
+
hydrated = true;
|
|
8205
|
+
} else {
|
|
8206
|
+
warnings.push(
|
|
8207
|
+
`Sidecar blob for attachment ${ref.id} failed BLAKE3 integrity check (expected ${ref.checksum}); imported as reference-only.`
|
|
8208
|
+
);
|
|
8209
|
+
}
|
|
8210
|
+
}
|
|
8211
|
+
}
|
|
8212
|
+
}
|
|
8213
|
+
if (!hydrated) {
|
|
8214
|
+
importedAttachmentReferenceCount++;
|
|
8215
|
+
}
|
|
8216
|
+
}
|
|
7433
8217
|
}
|
|
7434
8218
|
counts.notes++;
|
|
7435
8219
|
report?.({ phase: "notes", done: index + 1, total: parsedNotes.length });
|
|
7436
8220
|
await maybeYield2(index + 1, batchSize);
|
|
7437
8221
|
}
|
|
7438
|
-
if (
|
|
8222
|
+
if (importedAttachmentReferenceCount > 0) {
|
|
7439
8223
|
warnings.push(
|
|
7440
|
-
`${
|
|
8224
|
+
`${importedAttachmentReferenceCount} attachment reference(s) across ${notesWithImportedAttachmentReferences} note(s) were imported as metadata only: these shard records carry extracted text plus attachment metadata but no matching byte-sidecar entry, so their BlobStore bytes are unavailable (getBlob() returns null). Export a self-contained shard (\`includeBlobs\` + a \`blobStore\`) and import with a \`blobStore\` to hydrate bytes. Tracking: #271 / server #1046.`
|
|
7441
8225
|
);
|
|
7442
8226
|
}
|
|
7443
8227
|
const totalSkos = parsedSkosSchemes.length + parsedSkosConcepts.length + parsedSkosRelations.length + parsedNoteSkosTags.length;
|
|
7444
8228
|
let doneSkos = 0;
|
|
7445
8229
|
report?.({ phase: "skos", done: doneSkos, total: totalSkos });
|
|
7446
8230
|
for (const scheme of parsedSkosSchemes) {
|
|
8231
|
+
if (await skipExisting("skos_schemes", "SELECT 1 FROM skos_scheme WHERE id = $1", [scheme.id])) {
|
|
8232
|
+
report?.({ phase: "skos", done: ++doneSkos, total: totalSkos });
|
|
8233
|
+
await maybeYield2(doneSkos, batchSize);
|
|
8234
|
+
continue;
|
|
8235
|
+
}
|
|
7447
8236
|
if (strategy === "replace") {
|
|
7448
8237
|
await tx.query(
|
|
7449
8238
|
`INSERT INTO skos_scheme (id, title, description, created_at, updated_at)
|
|
@@ -7463,6 +8252,11 @@ async function importShard(db, data, options) {
|
|
|
7463
8252
|
await maybeYield2(doneSkos, batchSize);
|
|
7464
8253
|
}
|
|
7465
8254
|
for (const concept of parsedSkosConcepts) {
|
|
8255
|
+
if (await skipExisting("skos_concepts", "SELECT 1 FROM skos_concept WHERE id = $1", [concept.id])) {
|
|
8256
|
+
report?.({ phase: "skos", done: ++doneSkos, total: totalSkos });
|
|
8257
|
+
await maybeYield2(doneSkos, batchSize);
|
|
8258
|
+
continue;
|
|
8259
|
+
}
|
|
7466
8260
|
const altLabels = JSON.stringify(concept.alt_labels ?? []);
|
|
7467
8261
|
if (strategy === "replace") {
|
|
7468
8262
|
await tx.query(
|
|
@@ -7485,7 +8279,34 @@ async function importShard(db, data, options) {
|
|
|
7485
8279
|
report?.({ phase: "links", done: 0, total: parsedLinks.length });
|
|
7486
8280
|
for (const [index, shardLink] of parsedLinks.entries()) {
|
|
7487
8281
|
const link = linkFromShard(shardLink);
|
|
7488
|
-
if (
|
|
8282
|
+
if (!link.target_note_id) {
|
|
8283
|
+
if (!link.to_url) {
|
|
8284
|
+
skipped.links = (skipped.links ?? 0) + 1;
|
|
8285
|
+
warnings.push(`Shard link skipped: ${link.id} has neither to_note_id nor to_url.`);
|
|
8286
|
+
report?.({ phase: "links", done: index + 1, total: parsedLinks.length });
|
|
8287
|
+
await maybeYield2(index + 1, batchSize);
|
|
8288
|
+
continue;
|
|
8289
|
+
}
|
|
8290
|
+
const metadata = link.metadata == null ? null : JSON.stringify(link.metadata);
|
|
8291
|
+
await tx.query(
|
|
8292
|
+
`INSERT INTO link_url_target (
|
|
8293
|
+
id, source_note_id, to_url, link_type, confidence, metadata_json, created_at
|
|
8294
|
+
)
|
|
8295
|
+
VALUES ($1, $2, $3, $4, $5, $6::jsonb, $7)
|
|
8296
|
+
${strategy === "replace" ? "ON CONFLICT (id) DO UPDATE SET source_note_id = $2, to_url = $3, link_type = $4, confidence = $5, metadata_json = $6::jsonb, created_at = $7" : conflictClause}`,
|
|
8297
|
+
[link.id, link.source_note_id, link.to_url, link.link_type, link.confidence, metadata, link.created_at]
|
|
8298
|
+
);
|
|
8299
|
+
counts.links++;
|
|
8300
|
+
report?.({ phase: "links", done: index + 1, total: parsedLinks.length });
|
|
8301
|
+
await maybeYield2(index + 1, batchSize);
|
|
8302
|
+
continue;
|
|
8303
|
+
}
|
|
8304
|
+
if (await skipExisting("links", "SELECT 1 FROM link WHERE id = $1", [link.id])) {
|
|
8305
|
+
report?.({ phase: "links", done: index + 1, total: parsedLinks.length });
|
|
8306
|
+
await maybeYield2(index + 1, batchSize);
|
|
8307
|
+
continue;
|
|
8308
|
+
}
|
|
8309
|
+
if (strategy === "replace") {
|
|
7489
8310
|
await tx.query(
|
|
7490
8311
|
`INSERT INTO link (id, source_note_id, target_note_id, link_type, confidence, created_at)
|
|
7491
8312
|
VALUES ($1, $2, $3, $4, $5, $6)
|
|
@@ -7504,6 +8325,11 @@ async function importShard(db, data, options) {
|
|
|
7504
8325
|
await maybeYield2(index + 1, batchSize);
|
|
7505
8326
|
}
|
|
7506
8327
|
for (const relation of parsedSkosRelations) {
|
|
8328
|
+
if (await skipExisting("skos_relations", "SELECT 1 FROM skos_concept_relation WHERE id = $1", [relation.id])) {
|
|
8329
|
+
report?.({ phase: "skos", done: ++doneSkos, total: totalSkos });
|
|
8330
|
+
await maybeYield2(doneSkos, batchSize);
|
|
8331
|
+
continue;
|
|
8332
|
+
}
|
|
7507
8333
|
if (strategy === "replace") {
|
|
7508
8334
|
await tx.query(
|
|
7509
8335
|
`INSERT INTO skos_concept_relation (id, source_concept_id, target_concept_id, relation_type, created_at)
|
|
@@ -7523,6 +8349,11 @@ async function importShard(db, data, options) {
|
|
|
7523
8349
|
await maybeYield2(doneSkos, batchSize);
|
|
7524
8350
|
}
|
|
7525
8351
|
for (const tag of parsedNoteSkosTags) {
|
|
8352
|
+
if (await skipExisting("note_skos_tags", "SELECT 1 FROM note_skos_tag WHERE note_id = $1 AND concept_id = $2", [tag.note_id, tag.concept_id])) {
|
|
8353
|
+
report?.({ phase: "skos", done: ++doneSkos, total: totalSkos });
|
|
8354
|
+
await maybeYield2(doneSkos, batchSize);
|
|
8355
|
+
continue;
|
|
8356
|
+
}
|
|
7526
8357
|
await tx.query(
|
|
7527
8358
|
`INSERT INTO note_skos_tag (id, note_id, concept_id, created_at)
|
|
7528
8359
|
VALUES ($1, $2, $3, $4)
|
|
@@ -7535,6 +8366,11 @@ async function importShard(db, data, options) {
|
|
|
7535
8366
|
}
|
|
7536
8367
|
report?.({ phase: "provenance", done: 0, total: parsedProvenanceEdges.length });
|
|
7537
8368
|
for (const [index, edge] of parsedProvenanceEdges.entries()) {
|
|
8369
|
+
if (await skipExisting("provenance_edges", "SELECT 1 FROM provenance_edge WHERE id = $1", [edge.id])) {
|
|
8370
|
+
report?.({ phase: "provenance", done: index + 1, total: parsedProvenanceEdges.length });
|
|
8371
|
+
await maybeYield2(index + 1, batchSize);
|
|
8372
|
+
continue;
|
|
8373
|
+
}
|
|
7538
8374
|
const attributes = edge.attributes === null ? null : JSON.stringify(edge.attributes);
|
|
7539
8375
|
if (strategy === "replace") {
|
|
7540
8376
|
await tx.query(
|
|
@@ -7554,27 +8390,59 @@ async function importShard(db, data, options) {
|
|
|
7554
8390
|
report?.({ phase: "provenance", done: index + 1, total: parsedProvenanceEdges.length });
|
|
7555
8391
|
await maybeYield2(index + 1, batchSize);
|
|
7556
8392
|
}
|
|
8393
|
+
report?.({ phase: "embedding_configs", done: 0, total: parsedEmbConfigs.length });
|
|
8394
|
+
for (const [index, config] of parsedEmbConfigs.entries()) {
|
|
8395
|
+
await tx.query(
|
|
8396
|
+
`INSERT INTO embedding_config (
|
|
8397
|
+
id, name, description, model, dimension, chunk_size, chunk_overlap, is_default
|
|
8398
|
+
)
|
|
8399
|
+
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
|
|
8400
|
+
${strategy === "replace" ? "ON CONFLICT (id) DO UPDATE SET name = $2, description = $3, model = $4, dimension = $5, chunk_size = $6, chunk_overlap = $7, is_default = $8" : conflictClause}`,
|
|
8401
|
+
[
|
|
8402
|
+
config.id,
|
|
8403
|
+
config.name,
|
|
8404
|
+
config.description,
|
|
8405
|
+
config.model,
|
|
8406
|
+
config.dimension,
|
|
8407
|
+
config.chunk_size,
|
|
8408
|
+
config.chunk_overlap,
|
|
8409
|
+
config.is_default
|
|
8410
|
+
]
|
|
8411
|
+
);
|
|
8412
|
+
counts.embedding_configs++;
|
|
8413
|
+
report?.({ phase: "embedding_configs", done: index + 1, total: parsedEmbConfigs.length });
|
|
8414
|
+
await maybeYield2(index + 1, batchSize);
|
|
8415
|
+
}
|
|
7557
8416
|
report?.({ phase: "embedding_sets", done: 0, total: parsedEmbSets.length });
|
|
7558
8417
|
for (const [index, shardSet] of parsedEmbSets.entries()) {
|
|
7559
|
-
const set = embeddingSetFromShard(shardSet);
|
|
8418
|
+
const set = embeddingSetFromShard(shardSet, manifest.created_at);
|
|
8419
|
+
if (await skipExisting("embedding_sets", "SELECT 1 FROM embedding_set WHERE id = $1", [set.id])) {
|
|
8420
|
+
report?.({ phase: "embedding_sets", done: index + 1, total: parsedEmbSets.length });
|
|
8421
|
+
await maybeYield2(index + 1, batchSize);
|
|
8422
|
+
continue;
|
|
8423
|
+
}
|
|
7560
8424
|
if (strategy === "replace") {
|
|
7561
8425
|
await tx.query(
|
|
7562
8426
|
`INSERT INTO embedding_set (
|
|
7563
|
-
id, name,
|
|
8427
|
+
id, name, slug, description, purpose, document_count, embedding_count, is_system, keywords_json,
|
|
8428
|
+
model_name, dimensions, kind, mode, truncate_dimension,
|
|
7564
8429
|
criteria_json, source_json, compatibility_json, materialization_json, freshness_json, created_at, updated_at
|
|
7565
|
-
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::jsonb, $10::jsonb, $
|
|
7566
|
-
ON CONFLICT (id) DO UPDATE SET name = $2,
|
|
7567
|
-
|
|
7568
|
-
|
|
7569
|
-
|
|
8430
|
+
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::jsonb, $10, $11, $12, $13, $14, $15::jsonb, $16::jsonb, $17::jsonb, $18::jsonb, $19::jsonb, $20, COALESCE($21::timestamptz, $20::timestamptz))
|
|
8431
|
+
ON CONFLICT (id) DO UPDATE SET name = $2, slug = $3, description = $4, purpose = $5,
|
|
8432
|
+
document_count = $6, embedding_count = $7, is_system = $8, keywords_json = $9::jsonb,
|
|
8433
|
+
model_name = $10, dimensions = $11, kind = $12, mode = $13, truncate_dimension = $14,
|
|
8434
|
+
criteria_json = $15::jsonb, source_json = $16::jsonb, compatibility_json = $17::jsonb,
|
|
8435
|
+
materialization_json = $18::jsonb, freshness_json = $19::jsonb, updated_at = COALESCE($21::timestamptz, $20::timestamptz)`,
|
|
8436
|
+
[set.id, set.name, set.slug, set.description, set.purpose, set.document_count, set.embedding_count, set.is_system, set.keywords_json, set.model_name, set.dimensions, set.kind, set.mode, set.truncate_dimension, set.criteria_json, set.source_json, set.compatibility_json, set.materialization_json, set.freshness_json, set.created_at, set.updated_at]
|
|
7570
8437
|
);
|
|
7571
8438
|
} else {
|
|
7572
8439
|
await tx.query(
|
|
7573
8440
|
`INSERT INTO embedding_set (
|
|
7574
|
-
id, name,
|
|
8441
|
+
id, name, slug, description, purpose, document_count, embedding_count, is_system, keywords_json,
|
|
8442
|
+
model_name, dimensions, kind, mode, truncate_dimension,
|
|
7575
8443
|
criteria_json, source_json, compatibility_json, materialization_json, freshness_json, created_at, updated_at
|
|
7576
|
-
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::jsonb, $10::jsonb, $
|
|
7577
|
-
[set.id, set.name, set.purpose, set.model_name, set.dimensions, set.kind, set.mode, set.truncate_dimension, set.criteria_json, set.source_json, set.compatibility_json, set.materialization_json, set.freshness_json, set.created_at, set.updated_at]
|
|
8444
|
+
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::jsonb, $10, $11, $12, $13, $14, $15::jsonb, $16::jsonb, $17::jsonb, $18::jsonb, $19::jsonb, $20, COALESCE($21::timestamptz, $20::timestamptz)) ${conflictClause}`,
|
|
8445
|
+
[set.id, set.name, set.slug, set.description, set.purpose, set.document_count, set.embedding_count, set.is_system, set.keywords_json, set.model_name, set.dimensions, set.kind, set.mode, set.truncate_dimension, set.criteria_json, set.source_json, set.compatibility_json, set.materialization_json, set.freshness_json, set.created_at, set.updated_at]
|
|
7578
8446
|
);
|
|
7579
8447
|
}
|
|
7580
8448
|
counts.embedding_sets++;
|
|
@@ -7584,20 +8452,38 @@ async function importShard(db, data, options) {
|
|
|
7584
8452
|
report?.({ phase: "embeddings", done: 0, total: parsedEmbeddings.length });
|
|
7585
8453
|
for (const [index, shardEmb] of parsedEmbeddings.entries()) {
|
|
7586
8454
|
const emb = embeddingFromShard(shardEmb);
|
|
8455
|
+
const embeddingSetId = emb.embedding_set_id ?? await resolveEmbeddingSetIdForServerEmbedding(tx, emb.model, emb.vector);
|
|
8456
|
+
if (await skipExisting("embeddings", "SELECT 1 FROM embedding WHERE id = $1", [emb.id])) {
|
|
8457
|
+
report?.({ phase: "embeddings", done: index + 1, total: parsedEmbeddings.length });
|
|
8458
|
+
await maybeYield2(index + 1, batchSize);
|
|
8459
|
+
continue;
|
|
8460
|
+
}
|
|
7587
8461
|
if (strategy === "replace") {
|
|
7588
8462
|
await tx.query(
|
|
7589
|
-
`INSERT INTO embedding (id, note_id, embedding_set_id, vector, created_at)
|
|
7590
|
-
VALUES ($1, $2, $3, $4, $5)
|
|
7591
|
-
ON CONFLICT (id) DO UPDATE SET vector = $
|
|
7592
|
-
[emb.id, emb.note_id, emb.
|
|
8463
|
+
`INSERT INTO embedding (id, note_id, embedding_set_id, chunk_index, text, vector, model, created_at)
|
|
8464
|
+
VALUES ($1, $2, $3, $4, $5, $6, $7, COALESCE($8::timestamptz, now()))
|
|
8465
|
+
ON CONFLICT (id) DO UPDATE SET embedding_set_id = $3, chunk_index = $4, text = $5, vector = $6, model = $7`,
|
|
8466
|
+
[emb.id, emb.note_id, embeddingSetId, emb.chunk_index, emb.text, emb.vector, emb.model, emb.created_at]
|
|
7593
8467
|
);
|
|
7594
8468
|
} else {
|
|
7595
8469
|
await tx.query(
|
|
7596
|
-
`INSERT INTO embedding (id, note_id, embedding_set_id, vector, created_at)
|
|
7597
|
-
VALUES ($1, $2, $3, $4, $5) ${conflictClause}`,
|
|
7598
|
-
[emb.id, emb.note_id, emb.
|
|
8470
|
+
`INSERT INTO embedding (id, note_id, embedding_set_id, chunk_index, text, vector, model, created_at)
|
|
8471
|
+
VALUES ($1, $2, $3, $4, $5, $6, $7, COALESCE($8::timestamptz, now())) ${conflictClause}`,
|
|
8472
|
+
[emb.id, emb.note_id, embeddingSetId, emb.chunk_index, emb.text, emb.vector, emb.model, emb.created_at]
|
|
7599
8473
|
);
|
|
7600
8474
|
}
|
|
8475
|
+
await tx.query(
|
|
8476
|
+
`INSERT INTO embedding_set_member (
|
|
8477
|
+
embedding_set_id, note_id, embedding_id, membership_type, added_at, added_by
|
|
8478
|
+
) VALUES ($1, $2, $3, 'materialized', COALESCE($4::timestamptz, now()), 'shard-import')
|
|
8479
|
+
ON CONFLICT (embedding_set_id, note_id) DO UPDATE SET
|
|
8480
|
+
embedding_id = EXCLUDED.embedding_id,
|
|
8481
|
+
membership_type = CASE
|
|
8482
|
+
WHEN embedding_set_member.membership_type = 'auto' THEN 'materialized'
|
|
8483
|
+
ELSE embedding_set_member.membership_type
|
|
8484
|
+
END`,
|
|
8485
|
+
[embeddingSetId, emb.note_id, emb.id, emb.created_at]
|
|
8486
|
+
);
|
|
7601
8487
|
counts.embeddings++;
|
|
7602
8488
|
report?.({ phase: "embeddings", done: index + 1, total: parsedEmbeddings.length });
|
|
7603
8489
|
await maybeYield2(index + 1, batchSize);
|
|
@@ -7606,6 +8492,11 @@ async function importShard(db, data, options) {
|
|
|
7606
8492
|
let doneGraph = 0;
|
|
7607
8493
|
report?.({ phase: "graph", done: doneGraph, total: totalGraph });
|
|
7608
8494
|
for (const source of parsedGraphSources) {
|
|
8495
|
+
if (await skipExisting("graph_sources", "SELECT 1 FROM graph_source WHERE id = $1", [source.id])) {
|
|
8496
|
+
report?.({ phase: "graph", done: ++doneGraph, total: totalGraph });
|
|
8497
|
+
await maybeYield2(doneGraph, batchSize);
|
|
8498
|
+
continue;
|
|
8499
|
+
}
|
|
7609
8500
|
const parameters = source.parameters == null ? null : JSON.stringify(source.parameters);
|
|
7610
8501
|
const freshness = JSON.stringify({ ...source.freshness ?? {}, status: "unknown" });
|
|
7611
8502
|
await tx.query(
|
|
@@ -7621,6 +8512,15 @@ async function importShard(db, data, options) {
|
|
|
7621
8512
|
await maybeYield2(doneGraph, batchSize);
|
|
7622
8513
|
}
|
|
7623
8514
|
for (const edge of parsedGraphEdges) {
|
|
8515
|
+
if (await skipExisting(
|
|
8516
|
+
"graph_edges",
|
|
8517
|
+
"SELECT 1 FROM graph_edge_artifact WHERE graph_source_id = $1 AND from_note_id = $2 AND to_note_id = $3 AND kind = $4",
|
|
8518
|
+
[edge.graph_source_id, edge.from_note_id, edge.to_note_id, edge.kind]
|
|
8519
|
+
)) {
|
|
8520
|
+
report?.({ phase: "graph", done: ++doneGraph, total: totalGraph });
|
|
8521
|
+
await maybeYield2(doneGraph, batchSize);
|
|
8522
|
+
continue;
|
|
8523
|
+
}
|
|
7624
8524
|
const metadata = edge.metadata == null ? null : JSON.stringify(edge.metadata);
|
|
7625
8525
|
await tx.query(
|
|
7626
8526
|
`INSERT INTO graph_edge_artifact (graph_source_id, from_note_id, to_note_id, weight, kind, rank, metadata_json)
|
|
@@ -7638,16 +8538,22 @@ async function importShard(db, data, options) {
|
|
|
7638
8538
|
for (const set of parsedCommunitySets) {
|
|
7639
8539
|
const parameters = set.parameters == null ? null : JSON.stringify(set.parameters);
|
|
7640
8540
|
const freshness = JSON.stringify({ ...set.freshness ?? {}, status: "unknown" });
|
|
7641
|
-
await
|
|
8541
|
+
const skippedSet = await skipExisting("community_sets", "SELECT 1 FROM community_set WHERE id = $1", [set.id]);
|
|
8542
|
+
if (!skippedSet) await tx.query(
|
|
7642
8543
|
`INSERT INTO community_set (id, graph_source_id, name, source_type, algorithm, parameters_json, input_hash, freshness_json, created_at)
|
|
7643
8544
|
VALUES ($1, $2, $3, $4, $5, $6::jsonb, $7, $8::jsonb, $9)
|
|
7644
8545
|
${strategy === "replace" ? "ON CONFLICT (id) DO UPDATE SET graph_source_id = $2, name = $3, source_type = $4, algorithm = $5, parameters_json = $6::jsonb, input_hash = $7, freshness_json = $8::jsonb, created_at = $9" : conflictClause}`,
|
|
7645
8546
|
[set.id, set.graph_source_id, set.name, set.source_type, set.algorithm ?? null, parameters, set.input_hash, freshness, set.created_at]
|
|
7646
8547
|
);
|
|
7647
|
-
counts.community_sets++;
|
|
8548
|
+
if (!skippedSet) counts.community_sets++;
|
|
7648
8549
|
report?.({ phase: "communities", done: ++doneCommunities, total: totalCommunities });
|
|
7649
8550
|
await maybeYield2(doneCommunities, batchSize);
|
|
7650
8551
|
for (const community of set.communities ?? []) {
|
|
8552
|
+
if (await skipExisting("communities", "SELECT 1 FROM community WHERE community_set_id = $1 AND id = $2", [set.id, community.id])) {
|
|
8553
|
+
report?.({ phase: "communities", done: ++doneCommunities, total: totalCommunities });
|
|
8554
|
+
await maybeYield2(doneCommunities, batchSize);
|
|
8555
|
+
continue;
|
|
8556
|
+
}
|
|
7651
8557
|
const metadata = community.metadata == null ? null : JSON.stringify(community.metadata);
|
|
7652
8558
|
await tx.query(
|
|
7653
8559
|
`INSERT INTO community (community_set_id, id, label, rank, size, confidence, representative_note_ids, metadata_json)
|
|
@@ -7661,6 +8567,11 @@ async function importShard(db, data, options) {
|
|
|
7661
8567
|
}
|
|
7662
8568
|
}
|
|
7663
8569
|
for (const assignment of parsedCommunityAssignments) {
|
|
8570
|
+
if (await skipExisting("community_assignments", "SELECT 1 FROM community_assignment WHERE community_set_id = $1 AND note_id = $2", [assignment.community_set_id, assignment.note_id])) {
|
|
8571
|
+
report?.({ phase: "communities", done: ++doneCommunities, total: totalCommunities });
|
|
8572
|
+
await maybeYield2(doneCommunities, batchSize);
|
|
8573
|
+
continue;
|
|
8574
|
+
}
|
|
7664
8575
|
const metadata = assignment.metadata == null ? null : JSON.stringify(assignment.metadata);
|
|
7665
8576
|
await tx.query(
|
|
7666
8577
|
`INSERT INTO community_assignment (community_set_id, community_id, note_id, confidence, source_type, metadata_json)
|
|
@@ -7674,10 +8585,38 @@ async function importShard(db, data, options) {
|
|
|
7674
8585
|
}
|
|
7675
8586
|
report?.({ phase: "embedding_set_members", done: 0, total: parsedEmbMembers.length });
|
|
7676
8587
|
for (const [index, member] of parsedEmbMembers.entries()) {
|
|
8588
|
+
if (strategy === "skip" && preexistingEmbeddingMembers.has(`${member.embedding_set_id}\0${member.note_id}`)) {
|
|
8589
|
+
skipped.embedding_set_members = (skipped.embedding_set_members ?? 0) + 1;
|
|
8590
|
+
report?.({ phase: "embedding_set_members", done: index + 1, total: parsedEmbMembers.length });
|
|
8591
|
+
await maybeYield2(index + 1, batchSize);
|
|
8592
|
+
continue;
|
|
8593
|
+
}
|
|
8594
|
+
let embeddingId = member.embedding_id ?? null;
|
|
8595
|
+
if (!embeddingId) {
|
|
8596
|
+
const resolvedEmbedding = await tx.query(
|
|
8597
|
+
`SELECT id FROM embedding WHERE embedding_set_id = $1 AND note_id = $2 ORDER BY created_at DESC LIMIT 1`,
|
|
8598
|
+
[member.embedding_set_id, member.note_id]
|
|
8599
|
+
);
|
|
8600
|
+
embeddingId = resolvedEmbedding.rows[0]?.id ?? null;
|
|
8601
|
+
}
|
|
7677
8602
|
await tx.query(
|
|
7678
|
-
`INSERT INTO embedding_set_member (
|
|
7679
|
-
|
|
7680
|
-
|
|
8603
|
+
`INSERT INTO embedding_set_member (
|
|
8604
|
+
embedding_set_id, note_id, embedding_id, membership_type, added_at, added_by
|
|
8605
|
+
)
|
|
8606
|
+
VALUES ($1, $2, $3, $4, $5, $6)
|
|
8607
|
+
ON CONFLICT (embedding_set_id, note_id) DO UPDATE SET
|
|
8608
|
+
embedding_id = COALESCE(EXCLUDED.embedding_id, embedding_set_member.embedding_id),
|
|
8609
|
+
membership_type = EXCLUDED.membership_type,
|
|
8610
|
+
added_at = EXCLUDED.added_at,
|
|
8611
|
+
added_by = EXCLUDED.added_by`,
|
|
8612
|
+
[
|
|
8613
|
+
member.embedding_set_id,
|
|
8614
|
+
member.note_id,
|
|
8615
|
+
embeddingId,
|
|
8616
|
+
member.membership_type ?? "materialized",
|
|
8617
|
+
member.added_at ?? manifest.created_at,
|
|
8618
|
+
member.added_by ?? null
|
|
8619
|
+
]
|
|
7681
8620
|
);
|
|
7682
8621
|
counts.embedding_set_members++;
|
|
7683
8622
|
report?.({ phase: "embedding_set_members", done: index + 1, total: parsedEmbMembers.length });
|
|
@@ -7695,6 +8634,17 @@ async function importShard(db, data, options) {
|
|
|
7695
8634
|
duration_ms: performance.now() - start
|
|
7696
8635
|
};
|
|
7697
8636
|
}
|
|
8637
|
+
if (options?.blobStore && blobsToHydrate.size > 0) {
|
|
8638
|
+
try {
|
|
8639
|
+
for (const [hash, bytes] of blobsToHydrate) {
|
|
8640
|
+
await options.blobStore.write(hash, bytes);
|
|
8641
|
+
}
|
|
8642
|
+
} catch (err) {
|
|
8643
|
+
warnings.push(
|
|
8644
|
+
`Imported metadata successfully but failed to hydrate ${blobsToHydrate.size} attachment blob(s) into the BlobStore: ${err instanceof Error ? err.message : String(err)}.`
|
|
8645
|
+
);
|
|
8646
|
+
}
|
|
8647
|
+
}
|
|
7698
8648
|
return {
|
|
7699
8649
|
success: true,
|
|
7700
8650
|
counts,
|
|
@@ -7704,31 +8654,667 @@ async function importShard(db, data, options) {
|
|
|
7704
8654
|
duration_ms: performance.now() - start
|
|
7705
8655
|
};
|
|
7706
8656
|
}
|
|
7707
|
-
function
|
|
7708
|
-
|
|
7709
|
-
const
|
|
7710
|
-
|
|
8657
|
+
async function resolveEmbeddingSetIdForServerEmbedding(db, model, vector) {
|
|
8658
|
+
const dimension = vectorDimension(vector);
|
|
8659
|
+
const existing = await db.query(
|
|
8660
|
+
`SELECT id FROM embedding_set
|
|
8661
|
+
WHERE model_name = $1 AND dimensions = $2
|
|
8662
|
+
ORDER BY created_at, id
|
|
8663
|
+
LIMIT 1`,
|
|
8664
|
+
[model, dimension]
|
|
8665
|
+
);
|
|
8666
|
+
if (existing.rows[0]?.id) return existing.rows[0].id;
|
|
8667
|
+
const id = generateId();
|
|
8668
|
+
await db.query(
|
|
8669
|
+
`INSERT INTO embedding_set (
|
|
8670
|
+
id, name, slug, description, purpose, document_count, embedding_count,
|
|
8671
|
+
is_system, keywords_json, model_name, dimensions, kind, created_at, updated_at
|
|
8672
|
+
) VALUES ($1, $2, $3, NULL, NULL, 0, 0, false, '[]'::jsonb, $2, $4, 'physical', now(), now())`,
|
|
8673
|
+
[id, model, slugifyServerEmbeddingSet(model), dimension]
|
|
8674
|
+
);
|
|
8675
|
+
return id;
|
|
7711
8676
|
}
|
|
7712
|
-
function
|
|
7713
|
-
|
|
7714
|
-
|
|
8677
|
+
function vectorDimension(vector) {
|
|
8678
|
+
const inner = vector.replace(/^\[/, "").replace(/\]$/, "").trim();
|
|
8679
|
+
if (!inner) return 0;
|
|
8680
|
+
return inner.split(",").length;
|
|
8681
|
+
}
|
|
8682
|
+
function slugifyServerEmbeddingSet(value) {
|
|
8683
|
+
const slug = value.trim().toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "");
|
|
8684
|
+
return slug || "embedding-set";
|
|
8685
|
+
}
|
|
8686
|
+
|
|
8687
|
+
// schemas/knowledge-shard.schema.json
|
|
8688
|
+
var knowledge_shard_schema_default = {
|
|
8689
|
+
$schema: "https://json-schema.org/draft/2020-12/schema",
|
|
8690
|
+
$id: "https://fortemi.dev/schemas/knowledge-shard.schema.json",
|
|
8691
|
+
title: "Knowledge Shard (matric-shard) conformance schema",
|
|
8692
|
+
description: "Structural authority for the Knowledge Shard interchange contract shared between the Fortemi server (GET /api/v1/backup/knowledge-shard) and @fortemi/core. Covers the manifest, server-owned shard components, and React extension components tracked by issue #255.",
|
|
8693
|
+
type: "object",
|
|
8694
|
+
$defs: {
|
|
8695
|
+
isoDateTime: {
|
|
8696
|
+
type: "string",
|
|
8697
|
+
description: "ISO 8601 timestamp",
|
|
8698
|
+
minLength: 1
|
|
8699
|
+
},
|
|
8700
|
+
jsonObject: {
|
|
8701
|
+
type: "object",
|
|
8702
|
+
additionalProperties: true
|
|
8703
|
+
},
|
|
8704
|
+
nullableJsonObject: {
|
|
8705
|
+
type: ["object", "null"],
|
|
8706
|
+
additionalProperties: true
|
|
8707
|
+
},
|
|
8708
|
+
stringArray: {
|
|
8709
|
+
type: "array",
|
|
8710
|
+
items: { type: "string" }
|
|
8711
|
+
},
|
|
8712
|
+
manifest: {
|
|
8713
|
+
type: "object",
|
|
8714
|
+
additionalProperties: false,
|
|
8715
|
+
required: [
|
|
8716
|
+
"version",
|
|
8717
|
+
"matric_version",
|
|
8718
|
+
"format",
|
|
8719
|
+
"created_at",
|
|
8720
|
+
"components",
|
|
8721
|
+
"counts",
|
|
8722
|
+
"checksums",
|
|
8723
|
+
"min_reader_version"
|
|
8724
|
+
],
|
|
8725
|
+
properties: {
|
|
8726
|
+
version: { type: "string", minLength: 1 },
|
|
8727
|
+
matric_version: { type: "string", minLength: 1 },
|
|
8728
|
+
format: { const: "matric-shard" },
|
|
8729
|
+
created_at: { $ref: "#/$defs/isoDateTime" },
|
|
8730
|
+
components: {
|
|
8731
|
+
type: "array",
|
|
8732
|
+
items: {
|
|
8733
|
+
enum: [
|
|
8734
|
+
"notes",
|
|
8735
|
+
"collections",
|
|
8736
|
+
"tags",
|
|
8737
|
+
"templates",
|
|
8738
|
+
"links",
|
|
8739
|
+
"embedding_sets",
|
|
8740
|
+
"embedding_configs",
|
|
8741
|
+
"embedding_set_members",
|
|
8742
|
+
"embeddings",
|
|
8743
|
+
"skos_schemes",
|
|
8744
|
+
"skos_concepts",
|
|
8745
|
+
"skos_relations",
|
|
8746
|
+
"note_skos_tags",
|
|
8747
|
+
"provenance_edges",
|
|
8748
|
+
"community_assignments",
|
|
8749
|
+
"communities",
|
|
8750
|
+
"graph_edges",
|
|
8751
|
+
"graph_sources"
|
|
8752
|
+
]
|
|
8753
|
+
}
|
|
8754
|
+
},
|
|
8755
|
+
counts: {
|
|
8756
|
+
type: "object",
|
|
8757
|
+
additionalProperties: { type: "integer", minimum: 0 }
|
|
8758
|
+
},
|
|
8759
|
+
checksums: {
|
|
8760
|
+
type: "object",
|
|
8761
|
+
description: "filename -> sha256 hex",
|
|
8762
|
+
additionalProperties: { type: "string", pattern: "^[0-9a-f]{64}$" }
|
|
8763
|
+
},
|
|
8764
|
+
min_reader_version: { type: "string", minLength: 1 },
|
|
8765
|
+
migrated_from: {
|
|
8766
|
+
type: ["string", "null"],
|
|
8767
|
+
minLength: 1
|
|
8768
|
+
},
|
|
8769
|
+
migration_history: {
|
|
8770
|
+
type: "array",
|
|
8771
|
+
items: {
|
|
8772
|
+
type: "object",
|
|
8773
|
+
additionalProperties: false,
|
|
8774
|
+
required: [
|
|
8775
|
+
"from_version",
|
|
8776
|
+
"to_version",
|
|
8777
|
+
"migrated_at",
|
|
8778
|
+
"migrated_by",
|
|
8779
|
+
"changes"
|
|
8780
|
+
],
|
|
8781
|
+
properties: {
|
|
8782
|
+
from_version: { type: "string", minLength: 1 },
|
|
8783
|
+
to_version: { type: "string", minLength: 1 },
|
|
8784
|
+
migrated_at: { $ref: "#/$defs/isoDateTime" },
|
|
8785
|
+
migrated_by: { type: "string", minLength: 1 },
|
|
8786
|
+
changes: { $ref: "#/$defs/stringArray" }
|
|
8787
|
+
}
|
|
8788
|
+
}
|
|
8789
|
+
},
|
|
8790
|
+
layout: { type: "object" }
|
|
8791
|
+
}
|
|
8792
|
+
},
|
|
8793
|
+
attachmentReference: {
|
|
8794
|
+
type: "object",
|
|
8795
|
+
additionalProperties: false,
|
|
8796
|
+
required: ["id", "path", "mime", "checksum", "bytes"],
|
|
8797
|
+
properties: {
|
|
8798
|
+
id: { type: "string", minLength: 1 },
|
|
8799
|
+
path: { type: "string", minLength: 1 },
|
|
8800
|
+
mime: { type: ["string", "null"] },
|
|
8801
|
+
checksum: { type: "string", minLength: 1 },
|
|
8802
|
+
bytes: { type: "integer", minimum: 0 }
|
|
8803
|
+
}
|
|
8804
|
+
},
|
|
8805
|
+
attachmentProjection: {
|
|
8806
|
+
type: "object",
|
|
8807
|
+
additionalProperties: false,
|
|
8808
|
+
required: ["extracted_text", "attachment"],
|
|
8809
|
+
properties: {
|
|
8810
|
+
extracted_text: { type: ["string", "null"] },
|
|
8811
|
+
attachment: { $ref: "#/$defs/attachmentReference" }
|
|
8812
|
+
}
|
|
8813
|
+
},
|
|
8814
|
+
note: {
|
|
8815
|
+
type: "object",
|
|
8816
|
+
additionalProperties: false,
|
|
8817
|
+
description: "One row of the shard `notes` component. Field names are the server shard contract, not the PGlite table column names.",
|
|
8818
|
+
required: [
|
|
8819
|
+
"id",
|
|
8820
|
+
"title",
|
|
8821
|
+
"original_content",
|
|
8822
|
+
"revised_content",
|
|
8823
|
+
"format",
|
|
8824
|
+
"source",
|
|
8825
|
+
"starred",
|
|
8826
|
+
"archived",
|
|
8827
|
+
"tags",
|
|
8828
|
+
"created_at",
|
|
8829
|
+
"updated_at"
|
|
8830
|
+
],
|
|
8831
|
+
properties: {
|
|
8832
|
+
id: { type: "string", minLength: 1 },
|
|
8833
|
+
title: { type: ["string", "null"] },
|
|
8834
|
+
original_content: { type: "string" },
|
|
8835
|
+
revised_content: { type: ["string", "null"] },
|
|
8836
|
+
collection_id: { type: ["string", "null"] },
|
|
8837
|
+
attachments: {
|
|
8838
|
+
type: "array",
|
|
8839
|
+
items: { $ref: "#/$defs/attachmentProjection" }
|
|
8840
|
+
},
|
|
8841
|
+
format: { type: "string" },
|
|
8842
|
+
source: { type: "string" },
|
|
8843
|
+
starred: { type: "boolean" },
|
|
8844
|
+
archived: { type: "boolean" },
|
|
8845
|
+
tags: { type: "array", items: { type: "string" } },
|
|
8846
|
+
created_at: { $ref: "#/$defs/isoDateTime" },
|
|
8847
|
+
updated_at: { $ref: "#/$defs/isoDateTime" },
|
|
8848
|
+
deleted_at: { type: ["string", "null"] }
|
|
8849
|
+
}
|
|
8850
|
+
},
|
|
8851
|
+
collection: {
|
|
8852
|
+
type: "object",
|
|
8853
|
+
additionalProperties: false,
|
|
8854
|
+
required: ["id", "name", "description", "parent_id", "created_at"],
|
|
8855
|
+
properties: {
|
|
8856
|
+
id: { type: "string", minLength: 1 },
|
|
8857
|
+
name: { type: "string" },
|
|
8858
|
+
description: { type: ["string", "null"] },
|
|
8859
|
+
parent_id: { type: ["string", "null"] },
|
|
8860
|
+
created_at: { $ref: "#/$defs/isoDateTime" },
|
|
8861
|
+
note_count: { type: "integer", minimum: 0 }
|
|
8862
|
+
}
|
|
8863
|
+
},
|
|
8864
|
+
tag: {
|
|
8865
|
+
type: "object",
|
|
8866
|
+
additionalProperties: false,
|
|
8867
|
+
required: ["name", "created_at"],
|
|
8868
|
+
properties: {
|
|
8869
|
+
name: { type: "string", minLength: 1 },
|
|
8870
|
+
created_at: { $ref: "#/$defs/isoDateTime" }
|
|
8871
|
+
}
|
|
8872
|
+
},
|
|
8873
|
+
template: {
|
|
8874
|
+
type: "object",
|
|
8875
|
+
additionalProperties: false,
|
|
8876
|
+
required: ["id", "name", "description", "content", "format", "default_tags", "collection_id", "created_at", "updated_at"],
|
|
8877
|
+
properties: {
|
|
8878
|
+
id: { type: "string", minLength: 1 },
|
|
8879
|
+
name: { type: "string" },
|
|
8880
|
+
description: { type: ["string", "null"] },
|
|
8881
|
+
content: { type: "string" },
|
|
8882
|
+
format: { type: "string" },
|
|
8883
|
+
default_tags: { $ref: "#/$defs/stringArray" },
|
|
8884
|
+
collection_id: { type: ["string", "null"] },
|
|
8885
|
+
created_at: { $ref: "#/$defs/isoDateTime" },
|
|
8886
|
+
updated_at: { $ref: "#/$defs/isoDateTime" }
|
|
8887
|
+
}
|
|
8888
|
+
},
|
|
8889
|
+
link: {
|
|
8890
|
+
type: "object",
|
|
8891
|
+
additionalProperties: false,
|
|
8892
|
+
required: ["id", "from_note_id", "to_note_id", "to_url", "kind", "score", "created_at", "metadata"],
|
|
8893
|
+
properties: {
|
|
8894
|
+
id: { type: "string", minLength: 1 },
|
|
8895
|
+
from_note_id: { type: "string", minLength: 1 },
|
|
8896
|
+
to_note_id: { type: ["string", "null"] },
|
|
8897
|
+
to_url: { type: ["string", "null"] },
|
|
8898
|
+
kind: { type: "string" },
|
|
8899
|
+
score: { type: ["number", "null"] },
|
|
8900
|
+
created_at: { $ref: "#/$defs/isoDateTime" },
|
|
8901
|
+
metadata: { $ref: "#/$defs/nullableJsonObject" }
|
|
8902
|
+
}
|
|
8903
|
+
},
|
|
8904
|
+
embeddingSet: {
|
|
8905
|
+
type: "object",
|
|
8906
|
+
additionalProperties: false,
|
|
8907
|
+
required: ["id", "name", "slug", "description", "purpose", "document_count", "embedding_count", "is_system", "keywords", "model", "dimension"],
|
|
8908
|
+
properties: {
|
|
8909
|
+
id: { type: "string", minLength: 1 },
|
|
8910
|
+
name: { type: "string" },
|
|
8911
|
+
slug: { type: ["string", "null"] },
|
|
8912
|
+
description: { type: ["string", "null"] },
|
|
8913
|
+
purpose: { type: ["string", "null"] },
|
|
8914
|
+
document_count: { type: "integer", minimum: 0 },
|
|
8915
|
+
embedding_count: { type: "integer", minimum: 0 },
|
|
8916
|
+
is_system: { type: "boolean" },
|
|
8917
|
+
keywords: { $ref: "#/$defs/stringArray" },
|
|
8918
|
+
model: { type: "string" },
|
|
8919
|
+
dimension: { type: "integer", minimum: 1 },
|
|
8920
|
+
kind: { enum: ["physical", "filter", "virtual"] },
|
|
8921
|
+
mode: { type: ["string", "null"] },
|
|
8922
|
+
truncate_dimension: { type: ["integer", "null"], minimum: 1 },
|
|
8923
|
+
criteria: { $ref: "#/$defs/nullableJsonObject" },
|
|
8924
|
+
source: { $ref: "#/$defs/nullableJsonObject" },
|
|
8925
|
+
compatibility: { $ref: "#/$defs/nullableJsonObject" },
|
|
8926
|
+
materialization: { $ref: "#/$defs/nullableJsonObject" },
|
|
8927
|
+
freshness: { $ref: "#/$defs/nullableJsonObject" },
|
|
8928
|
+
created_at: { $ref: "#/$defs/isoDateTime" },
|
|
8929
|
+
updated_at: { $ref: "#/$defs/isoDateTime" }
|
|
8930
|
+
}
|
|
8931
|
+
},
|
|
8932
|
+
embeddingSetMember: {
|
|
8933
|
+
type: "object",
|
|
8934
|
+
additionalProperties: false,
|
|
8935
|
+
required: ["embedding_set_id", "note_id", "membership_type", "added_at", "added_by"],
|
|
8936
|
+
properties: {
|
|
8937
|
+
embedding_set_id: { type: "string", minLength: 1 },
|
|
8938
|
+
note_id: { type: "string", minLength: 1 },
|
|
8939
|
+
membership_type: { type: "string" },
|
|
8940
|
+
added_at: { $ref: "#/$defs/isoDateTime" },
|
|
8941
|
+
added_by: { type: ["string", "null"] }
|
|
8942
|
+
}
|
|
8943
|
+
},
|
|
8944
|
+
embeddingConfig: {
|
|
8945
|
+
type: "object",
|
|
8946
|
+
additionalProperties: false,
|
|
8947
|
+
required: ["id", "name", "description", "model", "dimension", "chunk_size", "chunk_overlap", "is_default"],
|
|
8948
|
+
properties: {
|
|
8949
|
+
id: { type: "string", minLength: 1 },
|
|
8950
|
+
name: { type: "string" },
|
|
8951
|
+
description: { type: ["string", "null"] },
|
|
8952
|
+
model: { type: "string" },
|
|
8953
|
+
dimension: { type: "integer", minimum: 1 },
|
|
8954
|
+
chunk_size: { type: "integer", minimum: 1 },
|
|
8955
|
+
chunk_overlap: { type: "integer", minimum: 0 },
|
|
8956
|
+
is_default: { type: "boolean" }
|
|
8957
|
+
}
|
|
8958
|
+
},
|
|
8959
|
+
embedding: {
|
|
8960
|
+
type: "object",
|
|
8961
|
+
additionalProperties: false,
|
|
8962
|
+
required: ["id", "note_id", "chunk_index", "text", "vector", "model"],
|
|
8963
|
+
properties: {
|
|
8964
|
+
id: { type: "string", minLength: 1 },
|
|
8965
|
+
note_id: { type: "string", minLength: 1 },
|
|
8966
|
+
chunk_index: { type: "integer", minimum: 0 },
|
|
8967
|
+
text: { type: "string" },
|
|
8968
|
+
vector: { type: "array", items: { type: "number" } },
|
|
8969
|
+
model: { type: "string" },
|
|
8970
|
+
embedding_set_id: { type: "string", minLength: 1 },
|
|
8971
|
+
created_at: { $ref: "#/$defs/isoDateTime" }
|
|
8972
|
+
}
|
|
8973
|
+
},
|
|
8974
|
+
skosScheme: {
|
|
8975
|
+
type: "object",
|
|
8976
|
+
additionalProperties: false,
|
|
8977
|
+
required: ["id", "title", "description", "created_at", "updated_at"],
|
|
8978
|
+
properties: {
|
|
8979
|
+
id: { type: "string", minLength: 1 },
|
|
8980
|
+
title: { type: "string" },
|
|
8981
|
+
description: { type: ["string", "null"] },
|
|
8982
|
+
created_at: { $ref: "#/$defs/isoDateTime" },
|
|
8983
|
+
updated_at: { $ref: "#/$defs/isoDateTime" }
|
|
8984
|
+
}
|
|
8985
|
+
},
|
|
8986
|
+
skosConcept: {
|
|
8987
|
+
type: "object",
|
|
8988
|
+
additionalProperties: false,
|
|
8989
|
+
required: ["id", "scheme_id", "pref_label", "alt_labels", "definition", "created_at", "updated_at"],
|
|
8990
|
+
properties: {
|
|
8991
|
+
id: { type: "string", minLength: 1 },
|
|
8992
|
+
scheme_id: { type: "string", minLength: 1 },
|
|
8993
|
+
pref_label: { type: "string" },
|
|
8994
|
+
alt_labels: { $ref: "#/$defs/stringArray" },
|
|
8995
|
+
definition: { type: ["string", "null"] },
|
|
8996
|
+
created_at: { $ref: "#/$defs/isoDateTime" },
|
|
8997
|
+
updated_at: { $ref: "#/$defs/isoDateTime" }
|
|
8998
|
+
}
|
|
8999
|
+
},
|
|
9000
|
+
skosRelation: {
|
|
9001
|
+
type: "object",
|
|
9002
|
+
additionalProperties: false,
|
|
9003
|
+
required: ["id", "source_concept_id", "target_concept_id", "relation_type", "created_at"],
|
|
9004
|
+
properties: {
|
|
9005
|
+
id: { type: "string", minLength: 1 },
|
|
9006
|
+
source_concept_id: { type: "string", minLength: 1 },
|
|
9007
|
+
target_concept_id: { type: "string", minLength: 1 },
|
|
9008
|
+
relation_type: { enum: ["broader", "narrower", "related"] },
|
|
9009
|
+
created_at: { $ref: "#/$defs/isoDateTime" }
|
|
9010
|
+
}
|
|
9011
|
+
},
|
|
9012
|
+
noteSkosTag: {
|
|
9013
|
+
type: "object",
|
|
9014
|
+
additionalProperties: false,
|
|
9015
|
+
required: ["id", "note_id", "concept_id", "created_at"],
|
|
9016
|
+
properties: {
|
|
9017
|
+
id: { type: "string", minLength: 1 },
|
|
9018
|
+
note_id: { type: "string", minLength: 1 },
|
|
9019
|
+
concept_id: { type: "string", minLength: 1 },
|
|
9020
|
+
created_at: { $ref: "#/$defs/isoDateTime" }
|
|
9021
|
+
}
|
|
9022
|
+
},
|
|
9023
|
+
provenanceEdge: {
|
|
9024
|
+
type: "object",
|
|
9025
|
+
additionalProperties: false,
|
|
9026
|
+
required: ["id", "entity_type", "entity_id", "activity", "agent", "started_at", "ended_at", "attributes"],
|
|
9027
|
+
properties: {
|
|
9028
|
+
id: { type: "string", minLength: 1 },
|
|
9029
|
+
entity_type: { type: "string" },
|
|
9030
|
+
entity_id: { type: "string", minLength: 1 },
|
|
9031
|
+
activity: { type: "string" },
|
|
9032
|
+
agent: { type: "string" },
|
|
9033
|
+
started_at: { $ref: "#/$defs/isoDateTime" },
|
|
9034
|
+
ended_at: { type: ["string", "null"] },
|
|
9035
|
+
attributes: { $ref: "#/$defs/nullableJsonObject" }
|
|
9036
|
+
}
|
|
9037
|
+
},
|
|
9038
|
+
artifactFreshness: {
|
|
9039
|
+
type: "object",
|
|
9040
|
+
additionalProperties: false,
|
|
9041
|
+
required: ["status"],
|
|
9042
|
+
properties: {
|
|
9043
|
+
status: { enum: ["fresh", "stale", "unknown"] },
|
|
9044
|
+
checked_at: { $ref: "#/$defs/isoDateTime" },
|
|
9045
|
+
stale_reason: { type: "string" },
|
|
9046
|
+
source_hashes: { $ref: "#/$defs/jsonObject" }
|
|
9047
|
+
}
|
|
9048
|
+
},
|
|
9049
|
+
graphSource: {
|
|
9050
|
+
type: "object",
|
|
9051
|
+
additionalProperties: false,
|
|
9052
|
+
required: ["id", "name", "kind", "input_hash", "freshness", "created_at"],
|
|
9053
|
+
properties: {
|
|
9054
|
+
id: { type: "string", minLength: 1 },
|
|
9055
|
+
name: { type: "string" },
|
|
9056
|
+
kind: { enum: ["link", "similarity", "search", "manual", "imported"] },
|
|
9057
|
+
source_table: { type: ["string", "null"] },
|
|
9058
|
+
embedding_set_id: { type: ["string", "null"] },
|
|
9059
|
+
virtual_set_id: { type: ["string", "null"] },
|
|
9060
|
+
model: { type: ["string", "null"] },
|
|
9061
|
+
dimension: { type: ["integer", "null"], minimum: 1 },
|
|
9062
|
+
truncate_dimension: { type: ["integer", "null"], minimum: 1 },
|
|
9063
|
+
metric: { type: ["string", "null"] },
|
|
9064
|
+
algorithm: { type: ["string", "null"] },
|
|
9065
|
+
parameters: { $ref: "#/$defs/jsonObject" },
|
|
9066
|
+
input_hash: { type: "string" },
|
|
9067
|
+
freshness: { $ref: "#/$defs/artifactFreshness" },
|
|
9068
|
+
created_at: { $ref: "#/$defs/isoDateTime" }
|
|
9069
|
+
}
|
|
9070
|
+
},
|
|
9071
|
+
graphEdge: {
|
|
9072
|
+
type: "object",
|
|
9073
|
+
additionalProperties: false,
|
|
9074
|
+
required: ["graph_source_id", "from_note_id", "to_note_id", "weight", "kind"],
|
|
9075
|
+
properties: {
|
|
9076
|
+
graph_source_id: { type: "string", minLength: 1 },
|
|
9077
|
+
from_note_id: { type: "string", minLength: 1 },
|
|
9078
|
+
to_note_id: { type: "string", minLength: 1 },
|
|
9079
|
+
weight: { type: "number" },
|
|
9080
|
+
kind: { enum: ["link", "similarity", "manual"] },
|
|
9081
|
+
rank: { type: ["integer", "null"], minimum: 0 },
|
|
9082
|
+
metadata: { $ref: "#/$defs/jsonObject" }
|
|
9083
|
+
}
|
|
9084
|
+
},
|
|
9085
|
+
community: {
|
|
9086
|
+
type: "object",
|
|
9087
|
+
additionalProperties: false,
|
|
9088
|
+
required: ["id"],
|
|
9089
|
+
properties: {
|
|
9090
|
+
id: { type: "string", minLength: 1 },
|
|
9091
|
+
label: { type: ["string", "null"] },
|
|
9092
|
+
rank: { type: ["integer", "null"], minimum: 0 },
|
|
9093
|
+
size: { type: ["integer", "null"], minimum: 0 },
|
|
9094
|
+
confidence: { type: ["number", "null"] },
|
|
9095
|
+
representative_note_ids: { $ref: "#/$defs/stringArray" },
|
|
9096
|
+
metadata: { $ref: "#/$defs/jsonObject" }
|
|
9097
|
+
}
|
|
9098
|
+
},
|
|
9099
|
+
communitySet: {
|
|
9100
|
+
type: "object",
|
|
9101
|
+
additionalProperties: false,
|
|
9102
|
+
required: ["id", "graph_source_id", "name", "source_type", "input_hash", "freshness", "communities", "created_at"],
|
|
9103
|
+
properties: {
|
|
9104
|
+
id: { type: "string", minLength: 1 },
|
|
9105
|
+
graph_source_id: { type: "string", minLength: 1 },
|
|
9106
|
+
name: { type: "string" },
|
|
9107
|
+
source_type: { enum: ["precomputed", "dynamic-snapshot", "user-authored", "imported"] },
|
|
9108
|
+
algorithm: { type: ["string", "null"] },
|
|
9109
|
+
parameters: { $ref: "#/$defs/jsonObject" },
|
|
9110
|
+
input_hash: { type: "string" },
|
|
9111
|
+
freshness: { $ref: "#/$defs/artifactFreshness" },
|
|
9112
|
+
communities: {
|
|
9113
|
+
type: "array",
|
|
9114
|
+
items: { $ref: "#/$defs/community" }
|
|
9115
|
+
},
|
|
9116
|
+
created_at: { $ref: "#/$defs/isoDateTime" }
|
|
9117
|
+
}
|
|
9118
|
+
},
|
|
9119
|
+
communityAssignment: {
|
|
9120
|
+
type: "object",
|
|
9121
|
+
additionalProperties: false,
|
|
9122
|
+
required: ["community_set_id", "community_id", "note_id", "source_type"],
|
|
9123
|
+
properties: {
|
|
9124
|
+
community_set_id: { type: "string", minLength: 1 },
|
|
9125
|
+
community_id: { type: "string", minLength: 1 },
|
|
9126
|
+
note_id: { type: "string", minLength: 1 },
|
|
9127
|
+
confidence: { type: ["number", "null"] },
|
|
9128
|
+
source_type: { enum: ["precomputed", "dynamic-snapshot", "user-authored", "imported"] },
|
|
9129
|
+
metadata: { $ref: "#/$defs/jsonObject" }
|
|
9130
|
+
}
|
|
9131
|
+
}
|
|
9132
|
+
}
|
|
9133
|
+
};
|
|
9134
|
+
|
|
9135
|
+
// src/shard/schema-validator.ts
|
|
9136
|
+
var decoder3 = new TextDecoder();
|
|
9137
|
+
var COMPONENT_SCHEMA_DEFS = {
|
|
9138
|
+
notes: "note",
|
|
9139
|
+
collections: "collection",
|
|
9140
|
+
tags: "tag",
|
|
9141
|
+
templates: "template",
|
|
9142
|
+
links: "link",
|
|
9143
|
+
embedding_sets: "embeddingSet",
|
|
9144
|
+
embedding_set_members: "embeddingSetMember",
|
|
9145
|
+
embedding_configs: "embeddingConfig",
|
|
9146
|
+
embeddings: "embedding",
|
|
9147
|
+
skos_schemes: "skosScheme",
|
|
9148
|
+
skos_concepts: "skosConcept",
|
|
9149
|
+
skos_relations: "skosRelation",
|
|
9150
|
+
note_skos_tags: "noteSkosTag",
|
|
9151
|
+
provenance_edges: "provenanceEdge",
|
|
9152
|
+
graph_sources: "graphSource",
|
|
9153
|
+
graph_edges: "graphEdge",
|
|
9154
|
+
communities: "communitySet",
|
|
9155
|
+
community_assignments: "communityAssignment"
|
|
9156
|
+
};
|
|
9157
|
+
var COMPONENT_FILES = {
|
|
9158
|
+
notes: { file: "notes.jsonl", encoding: "jsonl" },
|
|
9159
|
+
collections: { file: "collections.json", encoding: "json-array" },
|
|
9160
|
+
tags: { file: "tags.json", encoding: "json-array" },
|
|
9161
|
+
templates: { file: "templates.json", encoding: "json-array" },
|
|
9162
|
+
links: { file: "links.jsonl", encoding: "jsonl" },
|
|
9163
|
+
embedding_sets: { file: "embedding_sets.json", encoding: "json-array" },
|
|
9164
|
+
embedding_set_members: { file: "embedding_set_members.jsonl", encoding: "jsonl" },
|
|
9165
|
+
embedding_configs: { file: "embedding_configs.json", encoding: "json-array" },
|
|
9166
|
+
embeddings: { file: "embeddings.jsonl", encoding: "jsonl" },
|
|
9167
|
+
skos_schemes: { file: "skos_schemes.json", encoding: "json-array" },
|
|
9168
|
+
skos_concepts: { file: "skos_concepts.json", encoding: "json-array" },
|
|
9169
|
+
skos_relations: { file: "skos_relations.jsonl", encoding: "jsonl" },
|
|
9170
|
+
note_skos_tags: { file: "note_skos_tags.jsonl", encoding: "jsonl" },
|
|
9171
|
+
provenance_edges: { file: "provenance_edges.jsonl", encoding: "jsonl" },
|
|
9172
|
+
graph_sources: { file: "graph_sources.json", encoding: "json-array" },
|
|
9173
|
+
graph_edges: { file: "graph_edges.jsonl", encoding: "jsonl" },
|
|
9174
|
+
communities: { file: "communities.json", encoding: "json-array" },
|
|
9175
|
+
community_assignments: { file: "community_assignments.jsonl", encoding: "jsonl" }
|
|
9176
|
+
};
|
|
9177
|
+
var ajvInstance;
|
|
9178
|
+
var validators = /* @__PURE__ */ new Map();
|
|
9179
|
+
function getKnowledgeShardSchema() {
|
|
9180
|
+
return knowledge_shard_schema_default;
|
|
9181
|
+
}
|
|
9182
|
+
function getAjv() {
|
|
9183
|
+
if (!ajvInstance) {
|
|
9184
|
+
ajvInstance = new Ajv2020({
|
|
9185
|
+
allErrors: true,
|
|
9186
|
+
strict: true,
|
|
9187
|
+
validateFormats: false
|
|
9188
|
+
});
|
|
9189
|
+
ajvInstance.addSchema(knowledge_shard_schema_default);
|
|
9190
|
+
}
|
|
9191
|
+
return ajvInstance;
|
|
9192
|
+
}
|
|
9193
|
+
function validatorFor(defName) {
|
|
9194
|
+
const cached = validators.get(defName);
|
|
9195
|
+
if (cached) return cached;
|
|
9196
|
+
const validator = getAjv().getSchema(`${knowledge_shard_schema_default.$id}#/$defs/${defName}`) ?? getAjv().compile({ $ref: `${knowledge_shard_schema_default.$id}#/$defs/${defName}` });
|
|
9197
|
+
validators.set(defName, validator);
|
|
9198
|
+
return validator;
|
|
9199
|
+
}
|
|
9200
|
+
function formatErrors(errors) {
|
|
9201
|
+
return (errors ?? []).map((error) => {
|
|
9202
|
+
const path = error.instancePath || "(root)";
|
|
9203
|
+
return `${path} ${error.message ?? "is invalid"}`;
|
|
9204
|
+
});
|
|
9205
|
+
}
|
|
9206
|
+
function validateShardManifest(value) {
|
|
9207
|
+
const validate = validatorFor("manifest");
|
|
9208
|
+
const valid = validate(value);
|
|
9209
|
+
return { valid, errors: formatErrors(validate.errors) };
|
|
9210
|
+
}
|
|
9211
|
+
function parseJsonArray(bytes, path) {
|
|
9212
|
+
if (!bytes) return { records: [], errors: [] };
|
|
9213
|
+
try {
|
|
9214
|
+
const value = JSON.parse(decoder3.decode(bytes));
|
|
9215
|
+
if (!Array.isArray(value)) return { records: [], errors: [`${path} must be a JSON array`] };
|
|
9216
|
+
return { records: value, errors: [] };
|
|
9217
|
+
} catch (error) {
|
|
9218
|
+
return { records: [], errors: [`${path} failed to parse: ${error instanceof Error ? error.message : String(error)}`] };
|
|
9219
|
+
}
|
|
9220
|
+
}
|
|
9221
|
+
function parseJsonl(bytes, path) {
|
|
9222
|
+
if (!bytes) return { records: [], errors: [] };
|
|
9223
|
+
const text = decoder3.decode(bytes).trim();
|
|
9224
|
+
if (!text) return { records: [], errors: [] };
|
|
9225
|
+
const records = [];
|
|
9226
|
+
const errors = [];
|
|
9227
|
+
for (const [index, line] of text.split("\n").entries()) {
|
|
9228
|
+
try {
|
|
9229
|
+
records.push(JSON.parse(line));
|
|
9230
|
+
} catch (error) {
|
|
9231
|
+
errors.push(`${path}:${index + 1} failed to parse: ${error instanceof Error ? error.message : String(error)}`);
|
|
9232
|
+
}
|
|
9233
|
+
}
|
|
9234
|
+
return { records, errors };
|
|
9235
|
+
}
|
|
9236
|
+
function unpackShardFiles(input) {
|
|
9237
|
+
if (input instanceof Map) return input;
|
|
9238
|
+
return unpackTarGz(input instanceof ArrayBuffer ? new Uint8Array(input) : input);
|
|
9239
|
+
}
|
|
9240
|
+
function validateComponentRecords(component, records, path) {
|
|
9241
|
+
const errors = [];
|
|
9242
|
+
for (const [index, record] of records.entries()) {
|
|
9243
|
+
const result = validateShardComponentRecord(component, record);
|
|
9244
|
+
if (!result.valid) {
|
|
9245
|
+
errors.push(...result.errors.map((error) => `${path}[${index}] ${error}`));
|
|
9246
|
+
}
|
|
9247
|
+
}
|
|
9248
|
+
return errors;
|
|
9249
|
+
}
|
|
9250
|
+
function validateShardArchive(input) {
|
|
9251
|
+
let files;
|
|
9252
|
+
try {
|
|
9253
|
+
files = unpackShardFiles(input);
|
|
9254
|
+
} catch (error) {
|
|
9255
|
+
return {
|
|
9256
|
+
valid: false,
|
|
9257
|
+
errors: [`archive failed to unpack: ${error instanceof Error ? error.message : String(error)}`]
|
|
9258
|
+
};
|
|
9259
|
+
}
|
|
9260
|
+
const manifestBytes = files.get("manifest.json");
|
|
9261
|
+
if (!manifestBytes) return { valid: false, errors: ["manifest.json is missing"] };
|
|
9262
|
+
let manifest;
|
|
9263
|
+
try {
|
|
9264
|
+
manifest = JSON.parse(decoder3.decode(manifestBytes));
|
|
9265
|
+
} catch (error) {
|
|
9266
|
+
return {
|
|
9267
|
+
valid: false,
|
|
9268
|
+
errors: [`manifest.json failed to parse: ${error instanceof Error ? error.message : String(error)}`]
|
|
9269
|
+
};
|
|
9270
|
+
}
|
|
9271
|
+
const errors = [];
|
|
9272
|
+
const manifestResult = validateShardManifest(manifest);
|
|
9273
|
+
if (!manifestResult.valid) {
|
|
9274
|
+
errors.push(...manifestResult.errors.map((error) => `manifest.json ${error}`));
|
|
9275
|
+
}
|
|
9276
|
+
const componentsToValidate = new Set(manifest.components);
|
|
9277
|
+
for (const [component, spec] of Object.entries(COMPONENT_FILES)) {
|
|
9278
|
+
if (files.has(spec.file)) componentsToValidate.add(component);
|
|
9279
|
+
}
|
|
9280
|
+
for (const component of componentsToValidate) {
|
|
9281
|
+
const spec = COMPONENT_FILES[component];
|
|
9282
|
+
if (!spec) continue;
|
|
9283
|
+
if (component === "notes" && manifest.layout?.clusters?.notes?.length) {
|
|
9284
|
+
for (const cluster of manifest.layout.clusters.notes) {
|
|
9285
|
+
const parsed2 = parseJsonl(files.get(cluster.href), cluster.href);
|
|
9286
|
+
errors.push(...parsed2.errors);
|
|
9287
|
+
errors.push(...validateComponentRecords("notes", parsed2.records, cluster.href));
|
|
9288
|
+
}
|
|
9289
|
+
continue;
|
|
9290
|
+
}
|
|
9291
|
+
const parsed = spec.encoding === "json-array" ? parseJsonArray(files.get(spec.file), spec.file) : parseJsonl(files.get(spec.file), spec.file);
|
|
9292
|
+
errors.push(...parsed.errors);
|
|
9293
|
+
errors.push(...validateComponentRecords(component, parsed.records, spec.file));
|
|
9294
|
+
}
|
|
9295
|
+
return { valid: errors.length === 0, errors };
|
|
9296
|
+
}
|
|
9297
|
+
function validateShardComponentRecord(component, value) {
|
|
9298
|
+
const defName = COMPONENT_SCHEMA_DEFS[component];
|
|
9299
|
+
const validate = validatorFor(defName);
|
|
9300
|
+
const valid = validate(value);
|
|
9301
|
+
return { valid, errors: formatErrors(validate.errors) };
|
|
9302
|
+
}
|
|
9303
|
+
function assertShardComponentRecord(component, value) {
|
|
9304
|
+
const result = validateShardComponentRecord(component, value);
|
|
9305
|
+
if (!result.valid) {
|
|
9306
|
+
throw new Error(`Invalid shard ${component} record:
|
|
9307
|
+
${result.errors.join("\n")}`);
|
|
9308
|
+
}
|
|
7715
9309
|
}
|
|
7716
9310
|
|
|
7717
9311
|
// src/shard/shard-reader.ts
|
|
7718
|
-
var
|
|
9312
|
+
var decoder4 = new TextDecoder();
|
|
7719
9313
|
function assertSafeComponentName(filename) {
|
|
7720
9314
|
if (filename.length === 0 || filename.startsWith("/") || filename.includes("\\") || filename.includes("\0") || filename.includes(":") || filename.split("/").some((segment) => segment === "..")) {
|
|
7721
9315
|
throw new Error(`Refusing to read unsafe shard component path: ${JSON.stringify(filename)}`);
|
|
7722
9316
|
}
|
|
7723
9317
|
}
|
|
7724
|
-
function parseJsonlBytes(data) {
|
|
7725
|
-
if (!data || data.byteLength === 0) return [];
|
|
7726
|
-
return decoder2.decode(data).split("\n").filter((line) => line.trim().length > 0).map((line) => JSON.parse(line));
|
|
7727
|
-
}
|
|
7728
|
-
function parseJsonArrayBytes(data) {
|
|
7729
|
-
if (!data || data.byteLength === 0) return [];
|
|
7730
|
-
return JSON.parse(decoder2.decode(data));
|
|
7731
|
-
}
|
|
7732
9318
|
var DEFAULT_WEIGHTS = { title: 4, content: 1, tag: 2 };
|
|
7733
9319
|
async function toBytes(blobOrBytes) {
|
|
7734
9320
|
if (blobOrBytes instanceof Uint8Array) return blobOrBytes;
|
|
@@ -7741,8 +9327,13 @@ var PackedComponentStore = class {
|
|
|
7741
9327
|
this.files = files;
|
|
7742
9328
|
this.manifest = manifest;
|
|
7743
9329
|
}
|
|
7744
|
-
read(filename) {
|
|
7745
|
-
|
|
9330
|
+
async read(filename) {
|
|
9331
|
+
const bytes = this.files.get(filename);
|
|
9332
|
+
const expectedChecksum = this.manifest.checksums?.[filename];
|
|
9333
|
+
if (bytes && expectedChecksum && await sha256Hex(bytes) !== expectedChecksum) {
|
|
9334
|
+
throw new Error(`Checksum validation failed for shard component: ${filename}`);
|
|
9335
|
+
}
|
|
9336
|
+
return bytes;
|
|
7746
9337
|
}
|
|
7747
9338
|
};
|
|
7748
9339
|
var UrlComponentStore = class {
|
|
@@ -7750,10 +9341,12 @@ var UrlComponentStore = class {
|
|
|
7750
9341
|
baseUrl;
|
|
7751
9342
|
fetchImpl;
|
|
7752
9343
|
cache = /* @__PURE__ */ new Map();
|
|
7753
|
-
|
|
9344
|
+
maxComponentBytes;
|
|
9345
|
+
constructor(baseUrl, fetchImpl, manifest, maxComponentBytes) {
|
|
7754
9346
|
this.baseUrl = baseUrl.replace(/\/$/, "");
|
|
7755
9347
|
this.fetchImpl = fetchImpl;
|
|
7756
9348
|
this.manifest = manifest;
|
|
9349
|
+
this.maxComponentBytes = maxComponentBytes;
|
|
7757
9350
|
}
|
|
7758
9351
|
async read(filename) {
|
|
7759
9352
|
assertSafeComponentName(filename);
|
|
@@ -7763,12 +9356,47 @@ var UrlComponentStore = class {
|
|
|
7763
9356
|
this.cache.set(filename, void 0);
|
|
7764
9357
|
return void 0;
|
|
7765
9358
|
}
|
|
7766
|
-
const
|
|
9359
|
+
const declaredLength = Number(response.headers.get("content-length"));
|
|
9360
|
+
if (Number.isFinite(declaredLength) && declaredLength > this.maxComponentBytes) {
|
|
9361
|
+
throw new Error(`Shard component ${filename} exceeds cap ${this.maxComponentBytes} bytes`);
|
|
9362
|
+
}
|
|
9363
|
+
const chunks = [];
|
|
9364
|
+
let total = 0;
|
|
9365
|
+
if (response.body) {
|
|
9366
|
+
const reader = response.body.getReader();
|
|
9367
|
+
while (true) {
|
|
9368
|
+
const { done, value } = await reader.read();
|
|
9369
|
+
if (done) break;
|
|
9370
|
+
total += value.byteLength;
|
|
9371
|
+
if (total > this.maxComponentBytes) {
|
|
9372
|
+
await reader.cancel();
|
|
9373
|
+
throw new Error(`Shard component ${filename} exceeds cap ${this.maxComponentBytes} bytes`);
|
|
9374
|
+
}
|
|
9375
|
+
chunks.push(value);
|
|
9376
|
+
}
|
|
9377
|
+
} else {
|
|
9378
|
+
const value = new Uint8Array(await response.arrayBuffer());
|
|
9379
|
+
total = value.byteLength;
|
|
9380
|
+
if (total > this.maxComponentBytes) {
|
|
9381
|
+
throw new Error(`Shard component ${filename} exceeds cap ${this.maxComponentBytes} bytes`);
|
|
9382
|
+
}
|
|
9383
|
+
chunks.push(value);
|
|
9384
|
+
}
|
|
9385
|
+
const bytes = new Uint8Array(total);
|
|
9386
|
+
let offset = 0;
|
|
9387
|
+
for (const chunk of chunks) {
|
|
9388
|
+
bytes.set(chunk, offset);
|
|
9389
|
+
offset += chunk.byteLength;
|
|
9390
|
+
}
|
|
9391
|
+
const expectedChecksum = this.manifest.checksums?.[filename];
|
|
9392
|
+
if (expectedChecksum && await sha256Hex(bytes) !== expectedChecksum) {
|
|
9393
|
+
throw new Error(`Checksum validation failed for shard component: ${filename}`);
|
|
9394
|
+
}
|
|
7767
9395
|
this.cache.set(filename, bytes);
|
|
7768
9396
|
return bytes;
|
|
7769
9397
|
}
|
|
7770
9398
|
};
|
|
7771
|
-
async function resolveStore(source) {
|
|
9399
|
+
async function resolveStore(source, maxComponentBytes) {
|
|
7772
9400
|
if (typeof source === "object" && "baseUrl" in source) {
|
|
7773
9401
|
const fetchImpl = source.fetchImpl ?? globalThis.fetch;
|
|
7774
9402
|
const base = source.baseUrl.replace(/\/$/, "");
|
|
@@ -7777,20 +9405,20 @@ async function resolveStore(source) {
|
|
|
7777
9405
|
throw new Error(`Failed to fetch shard manifest (${manifestResponse.status}): ${base}/manifest.json`);
|
|
7778
9406
|
}
|
|
7779
9407
|
const manifest2 = await manifestResponse.json();
|
|
7780
|
-
return new UrlComponentStore(base, fetchImpl, manifest2);
|
|
9408
|
+
return new UrlComponentStore(base, fetchImpl, manifest2, maxComponentBytes);
|
|
7781
9409
|
}
|
|
7782
9410
|
const bytes = await toBytes(source);
|
|
7783
9411
|
const files = unpackTarGz(bytes);
|
|
7784
9412
|
const manifestBytes = files.get("manifest.json");
|
|
7785
9413
|
if (!manifestBytes) throw new Error("Missing manifest.json in shard archive");
|
|
7786
|
-
const manifest = JSON.parse(
|
|
9414
|
+
const manifest = JSON.parse(decoder4.decode(manifestBytes));
|
|
7787
9415
|
return new PackedComponentStore(files, manifest);
|
|
7788
9416
|
}
|
|
7789
9417
|
function tokenize(query) {
|
|
7790
9418
|
return query.toLowerCase().split(/[^a-z0-9]+/i).filter((token) => token.length > 0);
|
|
7791
9419
|
}
|
|
7792
9420
|
function noteSearchText(note) {
|
|
7793
|
-
const extractedText = note.binary_sources?.map((source) => source.extracted_text).filter(Boolean).join(" ") ?? "";
|
|
9421
|
+
const extractedText = (note.attachments ?? note.binary_sources)?.map((source) => source.extracted_text).filter(Boolean).join(" ") ?? "";
|
|
7794
9422
|
return `${note.title ?? ""} ${note.original_content} ${note.revised_content ?? ""} ${extractedText}`.toLowerCase();
|
|
7795
9423
|
}
|
|
7796
9424
|
function countOccurrences(haystack, needle) {
|
|
@@ -7812,7 +9440,7 @@ function noteMatchesTokens(note, tokens) {
|
|
|
7812
9440
|
function rankNote(note, tokens, weights) {
|
|
7813
9441
|
if (tokens.length === 0) return 0;
|
|
7814
9442
|
const title = (note.title ?? "").toLowerCase();
|
|
7815
|
-
const extractedText = note.binary_sources?.map((source) => source.extracted_text).filter(Boolean).join(" ") ?? "";
|
|
9443
|
+
const extractedText = (note.attachments ?? note.binary_sources)?.map((source) => source.extracted_text).filter(Boolean).join(" ") ?? "";
|
|
7816
9444
|
const content = `${note.original_content} ${note.revised_content ?? ""} ${extractedText}`.toLowerCase();
|
|
7817
9445
|
const tagText = note.tags.join(" ").toLowerCase();
|
|
7818
9446
|
let score = 0;
|
|
@@ -7824,7 +9452,7 @@ function rankNote(note, tokens, weights) {
|
|
|
7824
9452
|
return score;
|
|
7825
9453
|
}
|
|
7826
9454
|
function makeSnippet(note, tokens, length) {
|
|
7827
|
-
const extractedText = note.binary_sources?.map((source) => source.extracted_text).filter(Boolean).join(" ") ?? "";
|
|
9455
|
+
const extractedText = (note.attachments ?? note.binary_sources)?.map((source) => source.extracted_text).filter(Boolean).join(" ") ?? "";
|
|
7828
9456
|
const content = `${note.original_content} ${note.revised_content ?? ""} ${extractedText}`.trim();
|
|
7829
9457
|
if (tokens.length === 0) return content.slice(0, length);
|
|
7830
9458
|
const lower = content.toLowerCase();
|
|
@@ -8058,8 +9686,8 @@ var ShardReaderImpl = class {
|
|
|
8058
9686
|
}
|
|
8059
9687
|
};
|
|
8060
9688
|
async function openShard(source, options = {}) {
|
|
8061
|
-
const store = await resolveStore(source);
|
|
8062
|
-
if (store.manifest.min_reader_version && store.manifest.min_reader_version >
|
|
9689
|
+
const store = await resolveStore(source, options.maxComponentBytes ?? DEFAULT_MAX_DECOMPRESSED_BYTES);
|
|
9690
|
+
if (store.manifest.min_reader_version && compareShardVersions(store.manifest.min_reader_version, CURRENT_SHARD_VERSION) > 0) {
|
|
8063
9691
|
throw new Error(
|
|
8064
9692
|
`Shard requires reader version ${store.manifest.min_reader_version}, but this build supports ${CURRENT_SHARD_VERSION}. Import the shard instead.`
|
|
8065
9693
|
);
|
|
@@ -8068,7 +9696,7 @@ async function openShard(source, options = {}) {
|
|
|
8068
9696
|
}
|
|
8069
9697
|
|
|
8070
9698
|
// src/shard/semantic-providers.ts
|
|
8071
|
-
var
|
|
9699
|
+
var decoder5 = new TextDecoder();
|
|
8072
9700
|
function cosine(a, b) {
|
|
8073
9701
|
let dot = 0;
|
|
8074
9702
|
let normA = 0;
|
|
@@ -8092,7 +9720,7 @@ function createCosineSemanticProvider(options) {
|
|
|
8092
9720
|
vectors = [];
|
|
8093
9721
|
return;
|
|
8094
9722
|
}
|
|
8095
|
-
vectors =
|
|
9723
|
+
vectors = decoder5.decode(bytes).split("\n").filter((line) => line.trim().length > 0).map((line) => JSON.parse(line));
|
|
8096
9724
|
},
|
|
8097
9725
|
async search(query, k) {
|
|
8098
9726
|
const queryVector = await options.embedQuery(query);
|
|
@@ -8230,7 +9858,7 @@ var AIWG_SCAN_REQUIRED_FIELDS = [
|
|
|
8230
9858
|
];
|
|
8231
9859
|
function isPrivacyExcluded(record, options) {
|
|
8232
9860
|
const privacy = record.privacy;
|
|
8233
|
-
if (!privacy) return
|
|
9861
|
+
if (!privacy || !isPrivacyClassification(privacy.classification) || typeof privacy.pii !== "boolean") return true;
|
|
8234
9862
|
if (privacy.classification === "private" && !options?.includePrivate) return true;
|
|
8235
9863
|
if (privacy.pii && !options?.includePii) return true;
|
|
8236
9864
|
return false;
|
|
@@ -8299,6 +9927,10 @@ function validateOptionalRichMetadata(item, index, errors) {
|
|
|
8299
9927
|
errors.push("items[" + index + "].skos_concepts must be an array when present");
|
|
8300
9928
|
} else {
|
|
8301
9929
|
for (const [conceptIndex, concept] of item.skos_concepts.entries()) {
|
|
9930
|
+
if (!isPlainRecord(concept)) {
|
|
9931
|
+
errors.push("items[" + index + "].skos_concepts[" + conceptIndex + "] must be an object");
|
|
9932
|
+
continue;
|
|
9933
|
+
}
|
|
8302
9934
|
if (!hasString(concept.id)) errors.push("items[" + index + "].skos_concepts[" + conceptIndex + "].id is required");
|
|
8303
9935
|
if (!hasString(concept.prefLabel)) errors.push("items[" + index + "].skos_concepts[" + conceptIndex + "].prefLabel is required");
|
|
8304
9936
|
if (!isOptionalStringArray(concept.altLabels)) errors.push("items[" + index + "].skos_concepts[" + conceptIndex + "].altLabels must be a string array");
|
|
@@ -8313,6 +9945,10 @@ function validateOptionalRichMetadata(item, index, errors) {
|
|
|
8313
9945
|
errors.push("items[" + index + "].skos_relations must be an array when present");
|
|
8314
9946
|
} else {
|
|
8315
9947
|
for (const [relationIndex, relation] of item.skos_relations.entries()) {
|
|
9948
|
+
if (!isPlainRecord(relation)) {
|
|
9949
|
+
errors.push("items[" + index + "].skos_relations[" + relationIndex + "] must be an object");
|
|
9950
|
+
continue;
|
|
9951
|
+
}
|
|
8316
9952
|
if (!hasString(relation.type)) errors.push("items[" + index + "].skos_relations[" + relationIndex + "].type is required");
|
|
8317
9953
|
if (!hasString(relation.source_id)) errors.push("items[" + index + "].skos_relations[" + relationIndex + "].source_id is required");
|
|
8318
9954
|
if (!hasString(relation.target_id)) errors.push("items[" + index + "].skos_relations[" + relationIndex + "].target_id is required");
|
|
@@ -8327,6 +9963,10 @@ function validateOptionalRichMetadata(item, index, errors) {
|
|
|
8327
9963
|
errors.push("items[" + index + "].provenance_events must be an array when present");
|
|
8328
9964
|
} else {
|
|
8329
9965
|
for (const [eventIndex, event] of item.provenance_events.entries()) {
|
|
9966
|
+
if (!isPlainRecord(event)) {
|
|
9967
|
+
errors.push("items[" + index + "].provenance_events[" + eventIndex + "] must be an object");
|
|
9968
|
+
continue;
|
|
9969
|
+
}
|
|
8330
9970
|
if (!hasString(event.activity)) errors.push("items[" + index + "].provenance_events[" + eventIndex + "].activity is required");
|
|
8331
9971
|
if (event.attributes !== void 0 && !isPlainRecord(event.attributes)) {
|
|
8332
9972
|
errors.push("items[" + index + "].provenance_events[" + eventIndex + "].attributes must be an object");
|
|
@@ -8336,6 +9976,10 @@ function validateOptionalRichMetadata(item, index, errors) {
|
|
|
8336
9976
|
}
|
|
8337
9977
|
if (Array.isArray(item.relationships)) {
|
|
8338
9978
|
for (const [relationshipIndex, relationship] of item.relationships.entries()) {
|
|
9979
|
+
if (!isPlainRecord(relationship)) {
|
|
9980
|
+
errors.push("items[" + index + "].relationships[" + relationshipIndex + "] must be an object");
|
|
9981
|
+
continue;
|
|
9982
|
+
}
|
|
8339
9983
|
if (relationship.metadata !== void 0 && !isPlainRecord(relationship.metadata)) {
|
|
8340
9984
|
errors.push("items[" + index + "].relationships[" + relationshipIndex + "].metadata must be an object");
|
|
8341
9985
|
}
|
|
@@ -8364,7 +10008,10 @@ function validateOptionalRichMetadata(item, index, errors) {
|
|
|
8364
10008
|
errors.push("items[" + index + "].chunks must be an array when present");
|
|
8365
10009
|
} else {
|
|
8366
10010
|
for (const [chunkIndex, chunk] of item.chunks.entries()) {
|
|
8367
|
-
if (!isPlainRecord(chunk))
|
|
10011
|
+
if (!isPlainRecord(chunk)) {
|
|
10012
|
+
errors.push("items[" + index + "].chunks[" + chunkIndex + "] must be an object");
|
|
10013
|
+
continue;
|
|
10014
|
+
}
|
|
8368
10015
|
if (chunk.metadata !== void 0 && !isPlainRecord(chunk.metadata)) {
|
|
8369
10016
|
errors.push("items[" + index + "].chunks[" + chunkIndex + "].metadata must be an object");
|
|
8370
10017
|
}
|
|
@@ -8376,9 +10023,12 @@ function validateOptionalRichMetadata(item, index, errors) {
|
|
|
8376
10023
|
errors.push("items[" + index + "].embeddings must be an array when present");
|
|
8377
10024
|
} else {
|
|
8378
10025
|
for (const [embeddingIndex, embedding] of item.embeddings.entries()) {
|
|
8379
|
-
if (!isPlainRecord(embedding))
|
|
8380
|
-
|
|
8381
|
-
|
|
10026
|
+
if (!isPlainRecord(embedding)) {
|
|
10027
|
+
errors.push("items[" + index + "].embeddings[" + embeddingIndex + "] must be an object");
|
|
10028
|
+
continue;
|
|
10029
|
+
}
|
|
10030
|
+
const vector = embedding.embedding ?? embedding.vector;
|
|
10031
|
+
if (vector !== void 0 && (!Array.isArray(vector) || !vector.every((entry) => typeof entry === "number"))) {
|
|
8382
10032
|
errors.push("items[" + index + "].embeddings[" + embeddingIndex + "].embedding/vector must be a number array");
|
|
8383
10033
|
}
|
|
8384
10034
|
if (embedding.metadata !== void 0 && !isPlainRecord(embedding.metadata)) {
|
|
@@ -8412,7 +10062,15 @@ function validateProvenanceItems(item, index, errors) {
|
|
|
8412
10062
|
if (!isPrivacyClassification(prov.privacy)) errors.push(at + ".privacy must be one of private, sanitized, public");
|
|
8413
10063
|
}
|
|
8414
10064
|
}
|
|
8415
|
-
var V2_ONLY_RECORD_FIELDS = [
|
|
10065
|
+
var V2_ONLY_RECORD_FIELDS = [
|
|
10066
|
+
"search",
|
|
10067
|
+
"chunks",
|
|
10068
|
+
"embeddings",
|
|
10069
|
+
"skos_concepts",
|
|
10070
|
+
"skos_relations",
|
|
10071
|
+
"provenance_events",
|
|
10072
|
+
"compatibility"
|
|
10073
|
+
];
|
|
8416
10074
|
var V2_ONLY_SOURCE_FIELDS = ["origin", "generated", "checksum", "updated_at"];
|
|
8417
10075
|
var V2_ONLY_RELATIONSHIP_FIELDS = ["target_path", "direction", "metadata"];
|
|
8418
10076
|
function forbidV2FieldsOnV1Record(item, index, errors) {
|
|
@@ -8444,8 +10102,9 @@ function forbidV2FieldsOnV1Record(item, index, errors) {
|
|
|
8444
10102
|
}
|
|
8445
10103
|
function validateAiwgFortemiIndexExport(value) {
|
|
8446
10104
|
const errors = [];
|
|
8447
|
-
const counts =
|
|
8448
|
-
const data = value;
|
|
10105
|
+
const counts = /* @__PURE__ */ Object.create(null);
|
|
10106
|
+
const data = isPlainRecord(value) ? value : {};
|
|
10107
|
+
if (!isPlainRecord(value)) errors.push("index export must be an object");
|
|
8449
10108
|
if (!isSupportedIndexSchemaVersion(data?.schema_version)) {
|
|
8450
10109
|
errors.push("schema_version must be aiwg.fortemi.index.export.v1 or aiwg.fortemi.index.export.v2");
|
|
8451
10110
|
}
|
|
@@ -8469,17 +10128,25 @@ function validateAiwgFortemiIndexExport(value) {
|
|
|
8469
10128
|
}
|
|
8470
10129
|
const ids = /* @__PURE__ */ new Set();
|
|
8471
10130
|
let previousId = "";
|
|
8472
|
-
|
|
10131
|
+
const items = Array.isArray(data.items) ? data.items : [];
|
|
10132
|
+
for (const [index, item] of items.entries()) {
|
|
10133
|
+
if (!isPlainRecord(item)) {
|
|
10134
|
+
errors.push("items[" + index + "] must be an object");
|
|
10135
|
+
continue;
|
|
10136
|
+
}
|
|
8473
10137
|
for (const field of REQUIRED_RECORD_FIELDS) {
|
|
8474
10138
|
if (!(field in item)) errors.push("items[" + index + "]." + field + " is required");
|
|
8475
10139
|
}
|
|
8476
10140
|
if (!isSupportedRecordSchemaVersion(item.schema_version)) {
|
|
8477
10141
|
errors.push("items[" + index + "].schema_version must be aiwg.fortemi.index.record.v1 or aiwg.fortemi.index.record.v2");
|
|
8478
10142
|
}
|
|
10143
|
+
if (data.schema_version === "aiwg.fortemi.index.export.v1" && item.schema_version === "aiwg.fortemi.index.record.v2") {
|
|
10144
|
+
errors.push("items[" + index + "].schema_version must match aiwg.fortemi.index.export.v1");
|
|
10145
|
+
}
|
|
8479
10146
|
if (!hasString(item.id)) errors.push("items[" + index + "].id is required");
|
|
8480
10147
|
if (hasString(item.id) && ids.has(item.id)) errors.push("duplicate id: " + item.id);
|
|
8481
10148
|
if (hasString(item.id)) ids.add(item.id);
|
|
8482
|
-
if (previousId && hasString(item.id) && previousId
|
|
10149
|
+
if (previousId && hasString(item.id) && previousId > item.id) {
|
|
8483
10150
|
errors.push("items must be sorted by id: " + previousId + " before " + item.id);
|
|
8484
10151
|
}
|
|
8485
10152
|
if (hasString(item.id)) previousId = item.id;
|
|
@@ -8520,7 +10187,8 @@ function assertAiwgFortemiIndexExport(value) {
|
|
|
8520
10187
|
}
|
|
8521
10188
|
function validateAiwgFortemiChunkManifest(value) {
|
|
8522
10189
|
const errors = [];
|
|
8523
|
-
const data = value;
|
|
10190
|
+
const data = isPlainRecord(value) ? value : {};
|
|
10191
|
+
if (!isPlainRecord(value)) errors.push("chunk manifest must be an object");
|
|
8524
10192
|
if (data?.schema_version !== "aiwg.fortemi.index.chunk-manifest.v1") {
|
|
8525
10193
|
errors.push("schema_version must be aiwg.fortemi.index.chunk-manifest.v1");
|
|
8526
10194
|
}
|
|
@@ -8545,7 +10213,9 @@ function validateAiwgFortemiChunkManifest(value) {
|
|
|
8545
10213
|
}
|
|
8546
10214
|
}
|
|
8547
10215
|
}
|
|
8548
|
-
if (data.detail !== void 0) {
|
|
10216
|
+
if (data.detail !== void 0 && !isPlainRecord(data.detail)) {
|
|
10217
|
+
errors.push("detail must be an object");
|
|
10218
|
+
} else if (data.detail !== void 0) {
|
|
8549
10219
|
if (!hasString(data.detail.href)) errors.push("detail.href is required");
|
|
8550
10220
|
else if (!data.detail.href.includes("{id}")) errors.push("detail.href must contain the {id} placeholder");
|
|
8551
10221
|
if (data.detail.encoding !== void 0 && data.detail.encoding !== "uri" && data.detail.encoding !== "base64url") {
|
|
@@ -8556,6 +10226,10 @@ function validateAiwgFortemiChunkManifest(value) {
|
|
|
8556
10226
|
let expectedOffset = 0;
|
|
8557
10227
|
const parts = Array.isArray(data?.parts) ? data.parts : [];
|
|
8558
10228
|
for (const [index, part] of parts.entries()) {
|
|
10229
|
+
if (!isPlainRecord(part)) {
|
|
10230
|
+
errors.push("parts[" + index + "] must be an object");
|
|
10231
|
+
continue;
|
|
10232
|
+
}
|
|
8559
10233
|
if (!hasString(part.href)) errors.push("parts[" + index + "].href is required");
|
|
8560
10234
|
if (!hasNonNegativeInteger(part.offset)) errors.push("parts[" + index + "].offset must be a non-negative integer");
|
|
8561
10235
|
if (!hasNonNegativeInteger(part.count)) errors.push("parts[" + index + "].count must be a non-negative integer");
|
|
@@ -8576,18 +10250,54 @@ function assertAiwgFortemiChunkManifest(value) {
|
|
|
8576
10250
|
}
|
|
8577
10251
|
return value;
|
|
8578
10252
|
}
|
|
8579
|
-
function
|
|
10253
|
+
function validateProjectedRecordShape(item, index, errors) {
|
|
10254
|
+
const at = "items[" + index + "]";
|
|
10255
|
+
if (!item.privacy || !isPlainRecord(item.privacy)) {
|
|
10256
|
+
errors.push(at + "/privacy requires classification and pii");
|
|
10257
|
+
} else {
|
|
10258
|
+
if (!isPrivacyClassification(item.privacy.classification)) {
|
|
10259
|
+
errors.push(at + "/privacy/classification must be one of private, sanitized, public");
|
|
10260
|
+
}
|
|
10261
|
+
if (typeof item.privacy.pii !== "boolean") {
|
|
10262
|
+
errors.push(at + "/privacy/pii must be boolean");
|
|
10263
|
+
}
|
|
10264
|
+
}
|
|
10265
|
+
if (Array.isArray(item.provenance)) {
|
|
10266
|
+
for (const [provIndex, prov] of item.provenance.entries()) {
|
|
10267
|
+
if (!isPlainRecord(prov)) {
|
|
10268
|
+
errors.push(at + "/provenance/" + provIndex + " must be an object");
|
|
10269
|
+
continue;
|
|
10270
|
+
}
|
|
10271
|
+
if (!isProvenanceConfidence(prov.confidence)) {
|
|
10272
|
+
errors.push(at + "/provenance/" + provIndex + "/confidence must be one of source, candidate, reviewed, rejected");
|
|
10273
|
+
}
|
|
10274
|
+
if (!isPrivacyClassification(prov.privacy)) {
|
|
10275
|
+
errors.push(at + "/provenance/" + provIndex + "/privacy must be one of private, sanitized, public");
|
|
10276
|
+
}
|
|
10277
|
+
}
|
|
10278
|
+
}
|
|
10279
|
+
}
|
|
10280
|
+
function validateProjectedRecords(items, sourceSchemaVersion) {
|
|
8580
10281
|
const errors = [];
|
|
8581
10282
|
const ids = /* @__PURE__ */ new Set();
|
|
8582
10283
|
let previousId = "";
|
|
8583
10284
|
for (const [index, item] of items.entries()) {
|
|
10285
|
+
if (!isPlainRecord(item)) {
|
|
10286
|
+
errors.push("items[" + index + "] must be an object");
|
|
10287
|
+
continue;
|
|
10288
|
+
}
|
|
10289
|
+
validateProjectedRecordShape(item, index, errors);
|
|
10290
|
+
const expectedRecordVersion = sourceSchemaVersion === "aiwg.fortemi.index.export.v2" ? "aiwg.fortemi.index.record.v2" : "aiwg.fortemi.index.record.v1";
|
|
10291
|
+
if (item.schema_version !== expectedRecordVersion) {
|
|
10292
|
+
errors.push(`items[${index}].schema_version must match ${sourceSchemaVersion}`);
|
|
10293
|
+
}
|
|
8584
10294
|
if (!isSupportedRecordSchemaVersion(item.schema_version)) {
|
|
8585
10295
|
errors.push("items[" + index + "].schema_version must be aiwg.fortemi.index.record.v1 or aiwg.fortemi.index.record.v2");
|
|
8586
10296
|
}
|
|
8587
10297
|
if (!hasString(item.id)) errors.push("items[" + index + "].id is required");
|
|
8588
10298
|
if (hasString(item.id) && ids.has(item.id)) errors.push("duplicate id: " + item.id);
|
|
8589
10299
|
if (hasString(item.id)) ids.add(item.id);
|
|
8590
|
-
if (previousId && hasString(item.id) && previousId
|
|
10300
|
+
if (previousId && hasString(item.id) && previousId > item.id) {
|
|
8591
10301
|
errors.push("items must be sorted by id: " + previousId + " before " + item.id);
|
|
8592
10302
|
}
|
|
8593
10303
|
if (hasString(item.id)) previousId = item.id;
|
|
@@ -8603,15 +10313,13 @@ function validateProjectedRecords(items) {
|
|
|
8603
10313
|
}
|
|
8604
10314
|
if (!Array.isArray(item.tags)) errors.push("items[" + index + "].tags must be an array");
|
|
8605
10315
|
if (!Array.isArray(item.concepts)) errors.push("items[" + index + "].concepts must be an array");
|
|
8606
|
-
if (!item.privacy || !hasString(item.privacy.classification)) {
|
|
8607
|
-
errors.push("items[" + index + "].privacy.classification is required");
|
|
8608
|
-
}
|
|
8609
10316
|
}
|
|
8610
10317
|
return errors;
|
|
8611
10318
|
}
|
|
8612
10319
|
function validateAiwgFortemiChunkPart(value, partRef, manifest) {
|
|
8613
10320
|
const errors = [];
|
|
8614
|
-
const data = value;
|
|
10321
|
+
const data = isPlainRecord(value) ? value : {};
|
|
10322
|
+
if (!isPlainRecord(value)) errors.push("chunk part must be an object");
|
|
8615
10323
|
if (data?.schema_version !== "aiwg.fortemi.index.chunk.v1") {
|
|
8616
10324
|
errors.push("schema_version must be aiwg.fortemi.index.chunk.v1");
|
|
8617
10325
|
}
|
|
@@ -8628,12 +10336,16 @@ function validateAiwgFortemiChunkPart(value, partRef, manifest) {
|
|
|
8628
10336
|
}
|
|
8629
10337
|
if (Array.isArray(data?.items)) {
|
|
8630
10338
|
if (manifest?.projection) {
|
|
8631
|
-
errors.push(...validateProjectedRecords(
|
|
10339
|
+
errors.push(...validateProjectedRecords(
|
|
10340
|
+
data.items,
|
|
10341
|
+
manifest.source_export_schema_version ?? "aiwg.fortemi.index.export.v1"
|
|
10342
|
+
).map((error) => "items." + error));
|
|
8632
10343
|
} else {
|
|
8633
10344
|
const validation = validateAiwgFortemiIndexExport({
|
|
8634
|
-
schema_version: "aiwg.fortemi.index.export.v1",
|
|
10345
|
+
schema_version: manifest?.source_export_schema_version ?? "aiwg.fortemi.index.export.v1",
|
|
8635
10346
|
generated_at: manifest?.generated_at ?? "1970-01-01T00:00:00.000Z",
|
|
8636
10347
|
source: manifest?.source ?? { repo: "chunk", privacy: "public" },
|
|
10348
|
+
...(manifest?.source_export_schema_version ?? "aiwg.fortemi.index.export.v1") === "aiwg.fortemi.index.export.v2" ? { compatibility: { previous_schema_version: "aiwg.fortemi.index.export.v1", strategy: "supported" } } : {},
|
|
8637
10349
|
items: data.items
|
|
8638
10350
|
});
|
|
8639
10351
|
errors.push(...validation.errors.map((error) => "items." + error));
|
|
@@ -8720,15 +10432,17 @@ function recordTitle(item) {
|
|
|
8720
10432
|
}
|
|
8721
10433
|
function recordText(item) {
|
|
8722
10434
|
const base = item.text ?? item.search?.body ?? item.search?.summary ?? item.chunks?.map((chunk) => chunk.text ?? chunk.body ?? chunk.summary ?? "").filter(Boolean).join("\n") ?? "";
|
|
8723
|
-
const extractedText =
|
|
10435
|
+
const extractedText = binarySourceText(item);
|
|
8724
10436
|
return [base, extractedText].filter(Boolean).join("\n");
|
|
8725
10437
|
}
|
|
10438
|
+
function binarySourceText(item) {
|
|
10439
|
+
return "binary_sources" in item ? item.binary_sources?.map((source) => source.extracted_text).filter(Boolean).join("\n") ?? "" : "";
|
|
10440
|
+
}
|
|
8726
10441
|
function defaultEmbeddingInput(record, granularity) {
|
|
8727
10442
|
const title = recordTitle(record);
|
|
8728
10443
|
const text = recordText(record);
|
|
8729
10444
|
if (granularity === "title-summary") {
|
|
8730
|
-
|
|
8731
|
-
return [title, record.search?.summary ?? "", extractedText].filter(Boolean).join("\n");
|
|
10445
|
+
return [title, record.search?.summary ?? "", binarySourceText(record)].filter(Boolean).join("\n");
|
|
8732
10446
|
}
|
|
8733
10447
|
return [title, text].filter(Boolean).join("\n");
|
|
8734
10448
|
}
|
|
@@ -8736,6 +10450,137 @@ function generatedAtString(value) {
|
|
|
8736
10450
|
if (value instanceof Date) return value.toISOString();
|
|
8737
10451
|
return value ?? (/* @__PURE__ */ new Date()).toISOString();
|
|
8738
10452
|
}
|
|
10453
|
+
var SHA256_K = [
|
|
10454
|
+
1116352408,
|
|
10455
|
+
1899447441,
|
|
10456
|
+
3049323471,
|
|
10457
|
+
3921009573,
|
|
10458
|
+
961987163,
|
|
10459
|
+
1508970993,
|
|
10460
|
+
2453635748,
|
|
10461
|
+
2870763221,
|
|
10462
|
+
3624381080,
|
|
10463
|
+
310598401,
|
|
10464
|
+
607225278,
|
|
10465
|
+
1426881987,
|
|
10466
|
+
1925078388,
|
|
10467
|
+
2162078206,
|
|
10468
|
+
2614888103,
|
|
10469
|
+
3248222580,
|
|
10470
|
+
3835390401,
|
|
10471
|
+
4022224774,
|
|
10472
|
+
264347078,
|
|
10473
|
+
604807628,
|
|
10474
|
+
770255983,
|
|
10475
|
+
1249150122,
|
|
10476
|
+
1555081692,
|
|
10477
|
+
1996064986,
|
|
10478
|
+
2554220882,
|
|
10479
|
+
2821834349,
|
|
10480
|
+
2952996808,
|
|
10481
|
+
3210313671,
|
|
10482
|
+
3336571891,
|
|
10483
|
+
3584528711,
|
|
10484
|
+
113926993,
|
|
10485
|
+
338241895,
|
|
10486
|
+
666307205,
|
|
10487
|
+
773529912,
|
|
10488
|
+
1294757372,
|
|
10489
|
+
1396182291,
|
|
10490
|
+
1695183700,
|
|
10491
|
+
1986661051,
|
|
10492
|
+
2177026350,
|
|
10493
|
+
2456956037,
|
|
10494
|
+
2730485921,
|
|
10495
|
+
2820302411,
|
|
10496
|
+
3259730800,
|
|
10497
|
+
3345764771,
|
|
10498
|
+
3516065817,
|
|
10499
|
+
3600352804,
|
|
10500
|
+
4094571909,
|
|
10501
|
+
275423344,
|
|
10502
|
+
430227734,
|
|
10503
|
+
506948616,
|
|
10504
|
+
659060556,
|
|
10505
|
+
883997877,
|
|
10506
|
+
958139571,
|
|
10507
|
+
1322822218,
|
|
10508
|
+
1537002063,
|
|
10509
|
+
1747873779,
|
|
10510
|
+
1955562222,
|
|
10511
|
+
2024104815,
|
|
10512
|
+
2227730452,
|
|
10513
|
+
2361852424,
|
|
10514
|
+
2428436474,
|
|
10515
|
+
2756734187,
|
|
10516
|
+
3204031479,
|
|
10517
|
+
3329325298
|
|
10518
|
+
];
|
|
10519
|
+
function rotr(value, bits) {
|
|
10520
|
+
return value >>> bits | value << 32 - bits;
|
|
10521
|
+
}
|
|
10522
|
+
function sha256Hex2(data) {
|
|
10523
|
+
const bitLength = data.length * 8;
|
|
10524
|
+
const padded = new Uint8Array(data.length + 9 + 63 >> 6 << 6);
|
|
10525
|
+
padded.set(data);
|
|
10526
|
+
padded[data.length] = 128;
|
|
10527
|
+
const view = new DataView(padded.buffer);
|
|
10528
|
+
view.setUint32(padded.length - 4, bitLength >>> 0);
|
|
10529
|
+
view.setUint32(padded.length - 8, Math.floor(bitLength / 4294967296));
|
|
10530
|
+
let h0 = 1779033703;
|
|
10531
|
+
let h1 = 3144134277;
|
|
10532
|
+
let h2 = 1013904242;
|
|
10533
|
+
let h3 = 2773480762;
|
|
10534
|
+
let h4 = 1359893119;
|
|
10535
|
+
let h5 = 2600822924;
|
|
10536
|
+
let h6 = 528734635;
|
|
10537
|
+
let h7 = 1541459225;
|
|
10538
|
+
const w = new Uint32Array(64);
|
|
10539
|
+
for (let offset = 0; offset < padded.length; offset += 64) {
|
|
10540
|
+
for (let i = 0; i < 16; i++) w[i] = view.getUint32(offset + i * 4);
|
|
10541
|
+
for (let i = 16; i < 64; i++) {
|
|
10542
|
+
const s0 = rotr(w[i - 15], 7) ^ rotr(w[i - 15], 18) ^ w[i - 15] >>> 3;
|
|
10543
|
+
const s1 = rotr(w[i - 2], 17) ^ rotr(w[i - 2], 19) ^ w[i - 2] >>> 10;
|
|
10544
|
+
w[i] = w[i - 16] + s0 + w[i - 7] + s1 >>> 0;
|
|
10545
|
+
}
|
|
10546
|
+
let a = h0;
|
|
10547
|
+
let b = h1;
|
|
10548
|
+
let c = h2;
|
|
10549
|
+
let d = h3;
|
|
10550
|
+
let e = h4;
|
|
10551
|
+
let f = h5;
|
|
10552
|
+
let g = h6;
|
|
10553
|
+
let h = h7;
|
|
10554
|
+
for (let i = 0; i < 64; i++) {
|
|
10555
|
+
const s1 = rotr(e, 6) ^ rotr(e, 11) ^ rotr(e, 25);
|
|
10556
|
+
const ch = e & f ^ ~e & g;
|
|
10557
|
+
const temp1 = h + s1 + ch + SHA256_K[i] + w[i] >>> 0;
|
|
10558
|
+
const s0 = rotr(a, 2) ^ rotr(a, 13) ^ rotr(a, 22);
|
|
10559
|
+
const maj = a & b ^ a & c ^ b & c;
|
|
10560
|
+
const temp2 = s0 + maj >>> 0;
|
|
10561
|
+
h = g;
|
|
10562
|
+
g = f;
|
|
10563
|
+
f = e;
|
|
10564
|
+
e = d + temp1 >>> 0;
|
|
10565
|
+
d = c;
|
|
10566
|
+
c = b;
|
|
10567
|
+
b = a;
|
|
10568
|
+
a = temp1 + temp2 >>> 0;
|
|
10569
|
+
}
|
|
10570
|
+
h0 = h0 + a >>> 0;
|
|
10571
|
+
h1 = h1 + b >>> 0;
|
|
10572
|
+
h2 = h2 + c >>> 0;
|
|
10573
|
+
h3 = h3 + d >>> 0;
|
|
10574
|
+
h4 = h4 + e >>> 0;
|
|
10575
|
+
h5 = h5 + f >>> 0;
|
|
10576
|
+
h6 = h6 + g >>> 0;
|
|
10577
|
+
h7 = h7 + h >>> 0;
|
|
10578
|
+
}
|
|
10579
|
+
return [h0, h1, h2, h3, h4, h5, h6, h7].map((part) => part.toString(16).padStart(8, "0")).join("");
|
|
10580
|
+
}
|
|
10581
|
+
function computeAiwgIndexHash(data) {
|
|
10582
|
+
return `sha256:${sha256Hex2(data)}`;
|
|
10583
|
+
}
|
|
8739
10584
|
async function buildAiwgStaticEmbeddingSet(index, options) {
|
|
8740
10585
|
assertAiwgFortemiIndexExport(index);
|
|
8741
10586
|
const granularity = options.granularity ?? "body";
|
|
@@ -8751,7 +10596,7 @@ async function buildAiwgStaticEmbeddingSet(index, options) {
|
|
|
8751
10596
|
record_id: record.id,
|
|
8752
10597
|
embedding,
|
|
8753
10598
|
granularity,
|
|
8754
|
-
input_hash:
|
|
10599
|
+
input_hash: computeAiwgIndexHash(new TextEncoder().encode(input)),
|
|
8755
10600
|
source_path: record.source.path
|
|
8756
10601
|
});
|
|
8757
10602
|
}
|
|
@@ -8796,6 +10641,7 @@ function recordSearchValues(item) {
|
|
|
8796
10641
|
return values.filter((value) => typeof value === "string" && value.length > 0);
|
|
8797
10642
|
}
|
|
8798
10643
|
function buildAiwgChunkedIndex(index, options = {}) {
|
|
10644
|
+
assertAiwgFortemiIndexExport(index);
|
|
8799
10645
|
const partSize = hasPositiveInteger(options.partSize) ? options.partSize : 500;
|
|
8800
10646
|
const projection = options.projection;
|
|
8801
10647
|
const idEncoding = options.idEncoding ?? "base64url";
|
|
@@ -8875,35 +10721,88 @@ function queryMatches(item, q) {
|
|
|
8875
10721
|
return matches;
|
|
8876
10722
|
}
|
|
8877
10723
|
var DISCOVERY_STOPWORDS = /* @__PURE__ */ new Set([
|
|
10724
|
+
"the",
|
|
8878
10725
|
"a",
|
|
8879
10726
|
"an",
|
|
8880
10727
|
"and",
|
|
8881
|
-
"
|
|
8882
|
-
"
|
|
10728
|
+
"or",
|
|
10729
|
+
"of",
|
|
8883
10730
|
"for",
|
|
8884
|
-
"
|
|
8885
|
-
"how",
|
|
8886
|
-
"i",
|
|
10731
|
+
"to",
|
|
8887
10732
|
"in",
|
|
10733
|
+
"on",
|
|
10734
|
+
"with",
|
|
10735
|
+
"into",
|
|
10736
|
+
"from",
|
|
8888
10737
|
"is",
|
|
10738
|
+
"are",
|
|
10739
|
+
"be",
|
|
10740
|
+
"i",
|
|
10741
|
+
"we",
|
|
10742
|
+
"my",
|
|
10743
|
+
"it",
|
|
10744
|
+
"you",
|
|
8889
10745
|
"me",
|
|
8890
|
-
"
|
|
8891
|
-
"
|
|
8892
|
-
"
|
|
10746
|
+
"us",
|
|
10747
|
+
"your",
|
|
10748
|
+
"our",
|
|
10749
|
+
"this",
|
|
10750
|
+
"that",
|
|
10751
|
+
"these",
|
|
10752
|
+
"those",
|
|
10753
|
+
"there",
|
|
10754
|
+
"here",
|
|
10755
|
+
"some",
|
|
10756
|
+
"any",
|
|
10757
|
+
"all",
|
|
10758
|
+
"also",
|
|
8893
10759
|
"please",
|
|
8894
|
-
"
|
|
8895
|
-
"
|
|
8896
|
-
"
|
|
8897
|
-
"
|
|
10760
|
+
"about",
|
|
10761
|
+
"how",
|
|
10762
|
+
"what",
|
|
10763
|
+
"which",
|
|
10764
|
+
"where",
|
|
10765
|
+
"when",
|
|
10766
|
+
"who",
|
|
10767
|
+
"why",
|
|
10768
|
+
"find",
|
|
10769
|
+
"give",
|
|
10770
|
+
"show",
|
|
10771
|
+
"need",
|
|
10772
|
+
"want",
|
|
10773
|
+
"looking",
|
|
10774
|
+
"look",
|
|
10775
|
+
"help",
|
|
10776
|
+
"do",
|
|
10777
|
+
"does",
|
|
10778
|
+
"did",
|
|
10779
|
+
"can",
|
|
10780
|
+
"could",
|
|
10781
|
+
"should",
|
|
10782
|
+
"would",
|
|
10783
|
+
"will",
|
|
10784
|
+
"handle",
|
|
10785
|
+
"handles",
|
|
10786
|
+
"handling",
|
|
10787
|
+
"aiwg",
|
|
10788
|
+
"skill",
|
|
10789
|
+
"skills",
|
|
10790
|
+
"agent",
|
|
10791
|
+
"agents",
|
|
10792
|
+
"command",
|
|
10793
|
+
"commands",
|
|
10794
|
+
"rule",
|
|
10795
|
+
"rules",
|
|
10796
|
+
"flow",
|
|
10797
|
+
"flows",
|
|
10798
|
+
"workflow",
|
|
10799
|
+
"workflows"
|
|
8898
10800
|
]);
|
|
8899
|
-
function
|
|
8900
|
-
return value.toLowerCase().replace(/[_
|
|
8901
|
-
}
|
|
8902
|
-
function canonicalDiscoveryName(value) {
|
|
8903
|
-
return normalizeDiscoveryText(value).replace(/[\s.-]+/g, "");
|
|
10801
|
+
function normalizeDiscoveryName(value) {
|
|
10802
|
+
return value.toLowerCase().replace(/[-_\s]+/g, " ").trim();
|
|
8904
10803
|
}
|
|
8905
10804
|
function discoveryTokens(value) {
|
|
8906
|
-
return
|
|
10805
|
+
return value.toLowerCase().split(/[^a-z0-9-]+/).filter((token) => token.length > 1 && !DISCOVERY_STOPWORDS.has(token));
|
|
8907
10806
|
}
|
|
8908
10807
|
function facetValues(item, names) {
|
|
8909
10808
|
return names.flatMap((name) => item.facets[name] ?? []);
|
|
@@ -8913,17 +10812,58 @@ function addDiscoveryMatch(matches, match) {
|
|
|
8913
10812
|
matches.push(match);
|
|
8914
10813
|
}
|
|
8915
10814
|
}
|
|
8916
|
-
function
|
|
8917
|
-
if (
|
|
8918
|
-
|
|
8919
|
-
|
|
8920
|
-
|
|
10815
|
+
function damerauLevenshteinAtMostOne(left, right) {
|
|
10816
|
+
if (left === right) return true;
|
|
10817
|
+
if (Math.abs(left.length - right.length) > 1) return false;
|
|
10818
|
+
if (left.length === right.length) {
|
|
10819
|
+
let firstDiff = -1;
|
|
10820
|
+
let diffCount = 0;
|
|
10821
|
+
for (let index = 0; index < left.length; index += 1) {
|
|
10822
|
+
if (left[index] !== right[index]) {
|
|
10823
|
+
if (firstDiff < 0) firstDiff = index;
|
|
10824
|
+
diffCount += 1;
|
|
10825
|
+
}
|
|
10826
|
+
}
|
|
10827
|
+
if (diffCount === 1) return true;
|
|
10828
|
+
return diffCount === 2 && firstDiff + 1 < left.length && left[firstDiff] === right[firstDiff + 1] && left[firstDiff + 1] === right[firstDiff];
|
|
10829
|
+
}
|
|
10830
|
+
const shorter = left.length < right.length ? left : right;
|
|
10831
|
+
const longer = left.length < right.length ? right : left;
|
|
10832
|
+
let shorterIndex = 0;
|
|
10833
|
+
let longerIndex = 0;
|
|
10834
|
+
let edits = 0;
|
|
10835
|
+
while (shorterIndex < shorter.length && longerIndex < longer.length) {
|
|
10836
|
+
if (shorter[shorterIndex] === longer[longerIndex]) {
|
|
10837
|
+
shorterIndex += 1;
|
|
10838
|
+
longerIndex += 1;
|
|
10839
|
+
} else {
|
|
10840
|
+
edits += 1;
|
|
10841
|
+
if (edits > 1) return false;
|
|
10842
|
+
longerIndex += 1;
|
|
10843
|
+
}
|
|
10844
|
+
}
|
|
10845
|
+
return true;
|
|
10846
|
+
}
|
|
10847
|
+
function nearDiscoveryNameMatch(query, name) {
|
|
10848
|
+
const queryParts = normalizeDiscoveryName(query).split(/\s+/).filter(Boolean);
|
|
10849
|
+
const nameParts = normalizeDiscoveryName(name).split(/\s+/).filter(Boolean);
|
|
10850
|
+
if (queryParts.length !== nameParts.length) return false;
|
|
10851
|
+
return queryParts.every((part, index) => {
|
|
10852
|
+
const target = nameParts[index];
|
|
10853
|
+
if (part === target) return true;
|
|
10854
|
+
if (part.length < 5 || target.length < 5) return false;
|
|
10855
|
+
return damerauLevenshteinAtMostOne(part, target);
|
|
10856
|
+
});
|
|
10857
|
+
}
|
|
10858
|
+
function lowerValues(values) {
|
|
10859
|
+
return values.map((value) => value.toLowerCase());
|
|
8921
10860
|
}
|
|
8922
|
-
function discoveryMatches(item, query) {
|
|
10861
|
+
function discoveryMatches(item, query, options = {}) {
|
|
8923
10862
|
if (!query) return [];
|
|
8924
10863
|
const matches = [];
|
|
8925
10864
|
const tokens = discoveryTokens(query);
|
|
8926
|
-
const
|
|
10865
|
+
const lower = tokens.length > 0 ? tokens.join(" ") : query.toLowerCase().trim();
|
|
10866
|
+
const rawLower = query.toLowerCase().trim();
|
|
8927
10867
|
const idParts = item.id.split(/[:/]/);
|
|
8928
10868
|
const names = [
|
|
8929
10869
|
item.id,
|
|
@@ -8949,33 +10889,66 @@ function discoveryMatches(item, query) {
|
|
|
8949
10889
|
...item.tags
|
|
8950
10890
|
].filter((value) => hasString(value));
|
|
8951
10891
|
const sourceValues = [item.source?.path, item.source?.repo_relative_path, item.source?.locator].filter((value) => hasString(value));
|
|
10892
|
+
const title = recordTitle(item);
|
|
10893
|
+
const summary = [item.search?.summary, recordText(item)].filter((value) => hasString(value)).join("\n");
|
|
10894
|
+
const type = item.search?.type ?? item.type;
|
|
10895
|
+
const tags = [...item.search?.tags ?? [], ...item.tags];
|
|
10896
|
+
const useMultiToken = tokens.length > 1;
|
|
10897
|
+
const minHits = useMultiToken ? options.relaxOverlap ? 1 : Math.ceil(tokens.length / 2) : 1;
|
|
10898
|
+
const overlapOK = (hits) => useMultiToken && hits >= minHits;
|
|
10899
|
+
const addInfo = (match) => addDiscoveryMatch(matches, { ...match, score: 0 });
|
|
10900
|
+
const addScore = (score2) => {
|
|
10901
|
+
if (score2 > 0) addDiscoveryMatch(matches, { field: "id", value: item.id, score: score2, reason: "aiwg discovery score" });
|
|
10902
|
+
};
|
|
8952
10903
|
for (const name of names) {
|
|
8953
|
-
|
|
8954
|
-
|
|
8955
|
-
|
|
8956
|
-
|
|
8957
|
-
}
|
|
8958
|
-
|
|
10904
|
+
if (normalizeDiscoveryName(query) === normalizeDiscoveryName(name)) {
|
|
10905
|
+
addInfo({ field: "id", value: name, reason: "exact canonical name" });
|
|
10906
|
+
addScore(1.001);
|
|
10907
|
+
return matches;
|
|
10908
|
+
}
|
|
10909
|
+
if (nearDiscoveryNameMatch(query, name)) {
|
|
10910
|
+
addInfo({ field: "id", value: name, reason: "near canonical name" });
|
|
10911
|
+
addScore(0.951);
|
|
10912
|
+
return matches;
|
|
8959
10913
|
}
|
|
8960
10914
|
}
|
|
8961
|
-
|
|
8962
|
-
const
|
|
8963
|
-
|
|
8964
|
-
|
|
8965
|
-
|
|
8966
|
-
|
|
10915
|
+
let score = 0;
|
|
10916
|
+
const scoreText = (field, value, reason, exactScore, overlapScore, weight, exactBonus = 0) => {
|
|
10917
|
+
const normalized = value.toLowerCase();
|
|
10918
|
+
if (normalized.includes(lower)) {
|
|
10919
|
+
score += exactScore * weight;
|
|
10920
|
+
if (normalized === lower) score += exactBonus;
|
|
10921
|
+
addInfo({ field, value, reason });
|
|
10922
|
+
} else if (useMultiToken) {
|
|
10923
|
+
const hits = tokens.filter((token) => normalized.includes(token)).length;
|
|
10924
|
+
if (overlapOK(hits)) {
|
|
10925
|
+
score += overlapScore * weight * (hits / tokens.length);
|
|
10926
|
+
addInfo({ field, value, reason });
|
|
10927
|
+
}
|
|
10928
|
+
}
|
|
10929
|
+
};
|
|
10930
|
+
for (const trigger of lowerValues(triggers)) {
|
|
10931
|
+
if (trigger === lower || trigger === rawLower) {
|
|
10932
|
+
addInfo({ field: "facet", value: trigger, reason: "trigger phrase" });
|
|
10933
|
+
addScore(1.0008);
|
|
10934
|
+
return matches;
|
|
10935
|
+
}
|
|
8967
10936
|
}
|
|
8968
|
-
for (const
|
|
8969
|
-
|
|
8970
|
-
if (overlap > 0) addDiscoveryMatch(matches, { field: "concept", value: capability, score: 22 * overlap, reason: "capability overlap" });
|
|
10937
|
+
for (const trigger of triggers) {
|
|
10938
|
+
scoreText("facet", trigger, "trigger phrase", 0.25, 0.06, 4);
|
|
8971
10939
|
}
|
|
8972
|
-
const
|
|
8973
|
-
|
|
8974
|
-
|
|
10940
|
+
for (const capability of capabilities) scoreText("concept", capability, "capability overlap", 0.2, 0.1, 2);
|
|
10941
|
+
scoreText("title", title, "title token overlap", 0.3, 0.08, 3, 0.2);
|
|
10942
|
+
for (const tag of tags) scoreText("tag", tag, "tag token overlap", 0.2, 0.05, 2);
|
|
10943
|
+
if (summary) scoreText("text", summary, "body token overlap", 0.15, 0.04, 1);
|
|
8975
10944
|
for (const source of sourceValues) {
|
|
8976
|
-
|
|
8977
|
-
|
|
10945
|
+
scoreText("source", source, "path overlap", 0.1, 0.03, 1);
|
|
10946
|
+
}
|
|
10947
|
+
if (type.toLowerCase().includes(lower)) {
|
|
10948
|
+
score += 0.1;
|
|
10949
|
+
addInfo({ field: "facet", value: type, reason: "type overlap" });
|
|
8978
10950
|
}
|
|
10951
|
+
addScore(Math.min(score, 1));
|
|
8979
10952
|
return matches;
|
|
8980
10953
|
}
|
|
8981
10954
|
function rankMatches(matches, weights) {
|
|
@@ -9001,13 +10974,13 @@ function createSnippet(item, matches, q, maxLength) {
|
|
|
9001
10974
|
const firstMatch = textMatch ?? titleMatch ?? matches[0];
|
|
9002
10975
|
return clipSnippet(firstMatch?.value ?? recordText(item), q, maxLength);
|
|
9003
10976
|
}
|
|
9004
|
-
function createRankedEntries(items, q, options, ordinalBase = 0) {
|
|
10977
|
+
function createRankedEntries(items, q, options, ordinalBase = 0, discoveryOptions = {}) {
|
|
9005
10978
|
const weights = { ...DEFAULT_QUERY_WEIGHTS, ...options.weights };
|
|
9006
10979
|
const profile = options.searchProfile ?? "default";
|
|
9007
10980
|
return items.map((item, ordinal) => ({
|
|
9008
10981
|
item,
|
|
9009
10982
|
ordinal: ordinalBase + ordinal,
|
|
9010
|
-
matches: profile === "aiwg-discovery" ? discoveryMatches(item, q) : queryMatches(item, q)
|
|
10983
|
+
matches: profile === "aiwg-discovery" ? discoveryMatches(item, q, discoveryOptions) : queryMatches(item, q)
|
|
9011
10984
|
})).filter(({ item, matches }) => {
|
|
9012
10985
|
if (q && matches.length === 0) return false;
|
|
9013
10986
|
if (options.types && !options.types.includes(item.type)) return false;
|
|
@@ -9057,8 +11030,7 @@ function queryAiwgFortemiIndex(index, query = "", options = {}) {
|
|
|
9057
11030
|
const q = query.trim().toLowerCase();
|
|
9058
11031
|
const entries = createRankedEntries(index.items, q, options);
|
|
9059
11032
|
if (entries.length === 0 && q && options.searchProfile === "aiwg-discovery") {
|
|
9060
|
-
|
|
9061
|
-
return createQueryResultFromRankedEntries(createRankedEntries(index.items, relaxed, options), relaxed, options);
|
|
11033
|
+
return createQueryResultFromRankedEntries(createRankedEntries(index.items, q, options, 0, { relaxOverlap: true }), q, options);
|
|
9062
11034
|
}
|
|
9063
11035
|
return createQueryResultFromRankedEntries(entries, q, options);
|
|
9064
11036
|
}
|
|
@@ -9079,7 +11051,8 @@ function cosineSimilarity2(left, right) {
|
|
|
9079
11051
|
}
|
|
9080
11052
|
function validateAiwgStaticEmbeddingSet(value) {
|
|
9081
11053
|
const errors = [];
|
|
9082
|
-
const data = value;
|
|
11054
|
+
const data = isPlainRecord(value) ? value : {};
|
|
11055
|
+
if (!isPlainRecord(value)) errors.push("embedding set must be an object");
|
|
9083
11056
|
if (data?.schema_version !== "aiwg.fortemi.embedding.set.v1") errors.push("schema_version must be aiwg.fortemi.embedding.set.v1");
|
|
9084
11057
|
if (!hasString(data?.id)) errors.push("id is required");
|
|
9085
11058
|
if (!hasString(data?.model)) errors.push("model is required");
|
|
@@ -9087,7 +11060,12 @@ function validateAiwgStaticEmbeddingSet(value) {
|
|
|
9087
11060
|
if (!hasString(data?.generated_at)) errors.push("generated_at is required");
|
|
9088
11061
|
if (!hasString(data?.granularity)) errors.push("granularity is required");
|
|
9089
11062
|
if (!Array.isArray(data?.embeddings)) errors.push("embeddings must be an array");
|
|
9090
|
-
|
|
11063
|
+
const embeddings = Array.isArray(data.embeddings) ? data.embeddings : [];
|
|
11064
|
+
for (const [index, embedding] of embeddings.entries()) {
|
|
11065
|
+
if (!isPlainRecord(embedding)) {
|
|
11066
|
+
errors.push("embeddings[" + index + "] must be an object");
|
|
11067
|
+
continue;
|
|
11068
|
+
}
|
|
9091
11069
|
if (!hasString(embedding.record_id)) errors.push("embeddings[" + index + "].record_id is required");
|
|
9092
11070
|
if (!hasString(embedding.input_hash)) errors.push("embeddings[" + index + "].input_hash is required");
|
|
9093
11071
|
if (!Array.isArray(embedding.embedding)) errors.push("embeddings[" + index + "].embedding must be an array");
|
|
@@ -9117,7 +11095,10 @@ function queryAiwgSemanticIndex(index, embeddingSet, queryEmbedding, options = {
|
|
|
9117
11095
|
}).filter((result) => result !== null && result.score >= (options.minScore ?? -1)).sort((left, right) => right.score - left.score || left.item.id.localeCompare(right.item.id)).slice(offset, offset + limit);
|
|
9118
11096
|
}
|
|
9119
11097
|
function queryAiwgHybridIndex(index, embeddingSet, query, queryEmbedding, options = {}) {
|
|
9120
|
-
const
|
|
11098
|
+
const lexicalOptions = { ...options };
|
|
11099
|
+
delete lexicalOptions.limit;
|
|
11100
|
+
delete lexicalOptions.offset;
|
|
11101
|
+
const lexical = queryAiwgFortemiIndex(index, query, { ...lexicalOptions, rank: true });
|
|
9121
11102
|
const semantic = queryAiwgSemanticIndex(index, embeddingSet, queryEmbedding, { limit: index.items.length });
|
|
9122
11103
|
const lexicalWeight = options.lexicalWeight ?? 0.5;
|
|
9123
11104
|
const semanticWeight = options.semanticWeight ?? 0.5;
|
|
@@ -9187,6 +11168,7 @@ function matchSetCacheKey(q, options) {
|
|
|
9187
11168
|
concepts: options.concepts ?? null,
|
|
9188
11169
|
privacy: options.privacy ?? null,
|
|
9189
11170
|
rel: options.relationshipTargetId ?? null,
|
|
11171
|
+
searchProfile: options.searchProfile ?? "default",
|
|
9190
11172
|
weights: { ...DEFAULT_QUERY_WEIGHTS, ...options.weights }
|
|
9191
11173
|
});
|
|
9192
11174
|
}
|
|
@@ -9275,15 +11257,21 @@ function edgeFromRelationship(sourceId, relationship) {
|
|
|
9275
11257
|
}
|
|
9276
11258
|
function relationshipMatches(edge, options = {}) {
|
|
9277
11259
|
const type = relationshipTypeFilter(options);
|
|
9278
|
-
const direction = options.direction ?? "both";
|
|
9279
11260
|
if (type && edge.type !== type) return false;
|
|
9280
11261
|
if (options.relationshipDirection && edge.direction !== options.relationshipDirection) return false;
|
|
9281
11262
|
if (options.sourceId && edge.source_id !== options.sourceId) return false;
|
|
9282
11263
|
if (options.targetId && edge.target_id !== options.targetId) return false;
|
|
9283
|
-
if (
|
|
9284
|
-
if (direction === "in" && options.sourceId && edge.source_id !== options.sourceId) return false;
|
|
11264
|
+
if (options.endpointId && edge.source_id !== options.endpointId && edge.target_id !== options.endpointId) return false;
|
|
9285
11265
|
return true;
|
|
9286
11266
|
}
|
|
11267
|
+
function assertRelationshipDirectionAnchor(options) {
|
|
11268
|
+
if (options.direction === "out" && !options.sourceId) {
|
|
11269
|
+
throw new Error("relationship direction 'out' requires sourceId");
|
|
11270
|
+
}
|
|
11271
|
+
if (options.direction === "in" && !options.targetId) {
|
|
11272
|
+
throw new Error("relationship direction 'in' requires targetId");
|
|
11273
|
+
}
|
|
11274
|
+
}
|
|
9287
11275
|
function nodeSummary(item) {
|
|
9288
11276
|
return { id: item.id, type: item.type, title: recordTitle(item) };
|
|
9289
11277
|
}
|
|
@@ -9291,6 +11279,7 @@ function addNode(nodes, item) {
|
|
|
9291
11279
|
if (item) nodes.set(item.id, nodeSummary(item));
|
|
9292
11280
|
}
|
|
9293
11281
|
function relationshipResultFromRecords(records, options = {}) {
|
|
11282
|
+
assertRelationshipDirectionAnchor(options);
|
|
9294
11283
|
const byId = new Map(records.map((record) => [record.id, record]));
|
|
9295
11284
|
const edges = [];
|
|
9296
11285
|
for (const record of records) {
|
|
@@ -9300,7 +11289,8 @@ function relationshipResultFromRecords(records, options = {}) {
|
|
|
9300
11289
|
edges.push(edge);
|
|
9301
11290
|
}
|
|
9302
11291
|
}
|
|
9303
|
-
const
|
|
11292
|
+
const sortedEdges = edges.sort((left, right) => left.source_id.localeCompare(right.source_id) || left.target_id.localeCompare(right.target_id) || left.type.localeCompare(right.type));
|
|
11293
|
+
const limitedEdges = options.limit ? sortedEdges.slice(0, options.limit) : sortedEdges;
|
|
9304
11294
|
const nodes = /* @__PURE__ */ new Map();
|
|
9305
11295
|
for (const edge of limitedEdges) {
|
|
9306
11296
|
addNode(nodes, byId.get(edge.source_id));
|
|
@@ -9317,7 +11307,8 @@ function neighborQueryOptions(id, options = {}) {
|
|
|
9317
11307
|
return {
|
|
9318
11308
|
...options,
|
|
9319
11309
|
...direction === "out" ? { sourceId: id } : {},
|
|
9320
|
-
...direction === "in" ? { targetId: id } : {}
|
|
11310
|
+
...direction === "in" ? { targetId: id } : {},
|
|
11311
|
+
...direction === "both" ? { endpointId: id } : {}
|
|
9321
11312
|
};
|
|
9322
11313
|
}
|
|
9323
11314
|
function filterNeighborResult(id, result, options = {}) {
|
|
@@ -9626,13 +11617,14 @@ function createAiwgIndexController(initialIndex) {
|
|
|
9626
11617
|
}
|
|
9627
11618
|
function aiwgFortemiIndexToCommunityGraph(index, options = {}) {
|
|
9628
11619
|
const ids = new Set(index.items.map((item) => item.id));
|
|
9629
|
-
const relationshipWeights = options.relationshipWeights ??
|
|
11620
|
+
const relationshipWeights = options.relationshipWeights ?? /* @__PURE__ */ Object.create(null);
|
|
9630
11621
|
const edgeCounts = /* @__PURE__ */ new Map();
|
|
9631
11622
|
for (const item of index.items) {
|
|
9632
11623
|
for (const relationship of item.relationships) {
|
|
9633
11624
|
if (!ids.has(relationship.target_id) && !options.includeDanglingRelationships) continue;
|
|
9634
11625
|
const kind = relationship.type;
|
|
9635
|
-
const
|
|
11626
|
+
const configuredWeight = Object.prototype.hasOwnProperty.call(relationshipWeights, kind) ? relationshipWeights[kind] : void 0;
|
|
11627
|
+
const baseWeight = typeof configuredWeight === "number" && Number.isFinite(configuredWeight) ? configuredWeight : 1;
|
|
9636
11628
|
const key = `${item.id}\0${relationship.target_id}\0${kind}`;
|
|
9637
11629
|
const existing = edgeCounts.get(key);
|
|
9638
11630
|
if (existing) existing.weight += baseWeight;
|
|
@@ -9668,9 +11660,726 @@ function communityIdsFor(item, options) {
|
|
|
9668
11660
|
return [`type:${item.type}`];
|
|
9669
11661
|
}
|
|
9670
11662
|
|
|
11663
|
+
// schemas/aiwg-fortemi-index-export.schema.json
|
|
11664
|
+
var aiwg_fortemi_index_export_schema_default = {
|
|
11665
|
+
$schema: "https://json-schema.org/draft/2020-12/schema",
|
|
11666
|
+
$id: "https://aiwg.io/schemas/aiwg-fortemi-index-export.json",
|
|
11667
|
+
title: "AIWG Fortemi Index Export",
|
|
11668
|
+
type: "object",
|
|
11669
|
+
additionalProperties: false,
|
|
11670
|
+
required: ["schema_version", "generated_at", "source", "items"],
|
|
11671
|
+
properties: {
|
|
11672
|
+
schema_version: {
|
|
11673
|
+
enum: ["aiwg.fortemi.index.export.v1", "aiwg.fortemi.index.export.v2"]
|
|
11674
|
+
},
|
|
11675
|
+
generated_at: {
|
|
11676
|
+
type: "string",
|
|
11677
|
+
format: "date-time"
|
|
11678
|
+
},
|
|
11679
|
+
source: {
|
|
11680
|
+
type: "object",
|
|
11681
|
+
additionalProperties: false,
|
|
11682
|
+
required: ["repo", "privacy"],
|
|
11683
|
+
properties: {
|
|
11684
|
+
repo: {
|
|
11685
|
+
type: "string",
|
|
11686
|
+
minLength: 1
|
|
11687
|
+
},
|
|
11688
|
+
privacy: {
|
|
11689
|
+
$ref: "#/$defs/privacy"
|
|
11690
|
+
},
|
|
11691
|
+
graph: {
|
|
11692
|
+
type: "string",
|
|
11693
|
+
minLength: 1
|
|
11694
|
+
}
|
|
11695
|
+
}
|
|
11696
|
+
},
|
|
11697
|
+
compatibility: {
|
|
11698
|
+
type: "object",
|
|
11699
|
+
additionalProperties: false,
|
|
11700
|
+
required: ["previous_schema_version", "strategy"],
|
|
11701
|
+
properties: {
|
|
11702
|
+
previous_schema_version: {
|
|
11703
|
+
const: "aiwg.fortemi.index.export.v1"
|
|
11704
|
+
},
|
|
11705
|
+
strategy: {
|
|
11706
|
+
const: "supported"
|
|
11707
|
+
}
|
|
11708
|
+
}
|
|
11709
|
+
},
|
|
11710
|
+
items: {
|
|
11711
|
+
type: "array",
|
|
11712
|
+
items: {
|
|
11713
|
+
$ref: "#/$defs/record"
|
|
11714
|
+
}
|
|
11715
|
+
}
|
|
11716
|
+
},
|
|
11717
|
+
allOf: [
|
|
11718
|
+
{
|
|
11719
|
+
if: {
|
|
11720
|
+
properties: {
|
|
11721
|
+
schema_version: {
|
|
11722
|
+
const: "aiwg.fortemi.index.export.v1"
|
|
11723
|
+
}
|
|
11724
|
+
}
|
|
11725
|
+
},
|
|
11726
|
+
then: {
|
|
11727
|
+
not: {
|
|
11728
|
+
required: ["compatibility"]
|
|
11729
|
+
},
|
|
11730
|
+
properties: {
|
|
11731
|
+
source: {
|
|
11732
|
+
not: {
|
|
11733
|
+
required: ["graph"]
|
|
11734
|
+
}
|
|
11735
|
+
},
|
|
11736
|
+
items: {
|
|
11737
|
+
items: {
|
|
11738
|
+
allOf: [
|
|
11739
|
+
{
|
|
11740
|
+
properties: {
|
|
11741
|
+
schema_version: {
|
|
11742
|
+
const: "aiwg.fortemi.index.record.v1"
|
|
11743
|
+
}
|
|
11744
|
+
}
|
|
11745
|
+
},
|
|
11746
|
+
{
|
|
11747
|
+
$ref: "#/$defs/v1RecordCompatibility"
|
|
11748
|
+
}
|
|
11749
|
+
]
|
|
11750
|
+
}
|
|
11751
|
+
}
|
|
11752
|
+
}
|
|
11753
|
+
}
|
|
11754
|
+
},
|
|
11755
|
+
{
|
|
11756
|
+
if: {
|
|
11757
|
+
properties: {
|
|
11758
|
+
schema_version: {
|
|
11759
|
+
const: "aiwg.fortemi.index.export.v2"
|
|
11760
|
+
}
|
|
11761
|
+
}
|
|
11762
|
+
},
|
|
11763
|
+
then: {
|
|
11764
|
+
required: ["compatibility"],
|
|
11765
|
+
properties: {
|
|
11766
|
+
source: {
|
|
11767
|
+
required: ["graph"]
|
|
11768
|
+
},
|
|
11769
|
+
items: {
|
|
11770
|
+
items: {
|
|
11771
|
+
properties: {
|
|
11772
|
+
schema_version: {
|
|
11773
|
+
const: "aiwg.fortemi.index.record.v2"
|
|
11774
|
+
}
|
|
11775
|
+
}
|
|
11776
|
+
}
|
|
11777
|
+
}
|
|
11778
|
+
}
|
|
11779
|
+
}
|
|
11780
|
+
}
|
|
11781
|
+
],
|
|
11782
|
+
$defs: {
|
|
11783
|
+
privacy: {
|
|
11784
|
+
enum: ["private", "sanitized", "public"]
|
|
11785
|
+
},
|
|
11786
|
+
recordType: {
|
|
11787
|
+
type: "string",
|
|
11788
|
+
minLength: 1
|
|
11789
|
+
},
|
|
11790
|
+
stringArray: {
|
|
11791
|
+
type: "array",
|
|
11792
|
+
items: {
|
|
11793
|
+
type: "string"
|
|
11794
|
+
}
|
|
11795
|
+
},
|
|
11796
|
+
numberArray: {
|
|
11797
|
+
type: "array",
|
|
11798
|
+
items: {
|
|
11799
|
+
type: "number"
|
|
11800
|
+
}
|
|
11801
|
+
},
|
|
11802
|
+
v1RecordCompatibility: {
|
|
11803
|
+
type: "object",
|
|
11804
|
+
not: {
|
|
11805
|
+
anyOf: [
|
|
11806
|
+
{
|
|
11807
|
+
required: ["name"]
|
|
11808
|
+
},
|
|
11809
|
+
{
|
|
11810
|
+
required: ["summary"]
|
|
11811
|
+
},
|
|
11812
|
+
{
|
|
11813
|
+
required: ["search"]
|
|
11814
|
+
},
|
|
11815
|
+
{
|
|
11816
|
+
required: ["chunks"]
|
|
11817
|
+
},
|
|
11818
|
+
{
|
|
11819
|
+
required: ["embeddings"]
|
|
11820
|
+
},
|
|
11821
|
+
{
|
|
11822
|
+
required: ["compatibility"]
|
|
11823
|
+
},
|
|
11824
|
+
{
|
|
11825
|
+
required: ["skos_concepts"]
|
|
11826
|
+
},
|
|
11827
|
+
{
|
|
11828
|
+
required: ["skos_relations"]
|
|
11829
|
+
},
|
|
11830
|
+
{
|
|
11831
|
+
required: ["provenance_events"]
|
|
11832
|
+
}
|
|
11833
|
+
]
|
|
11834
|
+
},
|
|
11835
|
+
properties: {
|
|
11836
|
+
source: {
|
|
11837
|
+
not: {
|
|
11838
|
+
anyOf: [
|
|
11839
|
+
{
|
|
11840
|
+
required: ["origin"]
|
|
11841
|
+
},
|
|
11842
|
+
{
|
|
11843
|
+
required: ["generated"]
|
|
11844
|
+
},
|
|
11845
|
+
{
|
|
11846
|
+
required: ["checksum"]
|
|
11847
|
+
},
|
|
11848
|
+
{
|
|
11849
|
+
required: ["updated_at"]
|
|
11850
|
+
}
|
|
11851
|
+
]
|
|
11852
|
+
}
|
|
11853
|
+
},
|
|
11854
|
+
relationships: {
|
|
11855
|
+
items: {
|
|
11856
|
+
not: {
|
|
11857
|
+
anyOf: [
|
|
11858
|
+
{
|
|
11859
|
+
required: ["target_path"]
|
|
11860
|
+
},
|
|
11861
|
+
{
|
|
11862
|
+
required: ["direction"]
|
|
11863
|
+
},
|
|
11864
|
+
{
|
|
11865
|
+
required: ["label"]
|
|
11866
|
+
},
|
|
11867
|
+
{
|
|
11868
|
+
required: ["confidence"]
|
|
11869
|
+
},
|
|
11870
|
+
{
|
|
11871
|
+
required: ["privacy"]
|
|
11872
|
+
},
|
|
11873
|
+
{
|
|
11874
|
+
required: ["metadata"]
|
|
11875
|
+
}
|
|
11876
|
+
]
|
|
11877
|
+
}
|
|
11878
|
+
}
|
|
11879
|
+
},
|
|
11880
|
+
privacy: {
|
|
11881
|
+
not: {
|
|
11882
|
+
required: ["locality"]
|
|
11883
|
+
}
|
|
11884
|
+
}
|
|
11885
|
+
}
|
|
11886
|
+
},
|
|
11887
|
+
record: {
|
|
11888
|
+
type: "object",
|
|
11889
|
+
additionalProperties: false,
|
|
11890
|
+
required: [
|
|
11891
|
+
"schema_version",
|
|
11892
|
+
"id",
|
|
11893
|
+
"type",
|
|
11894
|
+
"source",
|
|
11895
|
+
"title",
|
|
11896
|
+
"text",
|
|
11897
|
+
"facets",
|
|
11898
|
+
"tags",
|
|
11899
|
+
"concepts",
|
|
11900
|
+
"relationships",
|
|
11901
|
+
"provenance",
|
|
11902
|
+
"privacy",
|
|
11903
|
+
"updated_at"
|
|
11904
|
+
],
|
|
11905
|
+
properties: {
|
|
11906
|
+
schema_version: {
|
|
11907
|
+
enum: [
|
|
11908
|
+
"aiwg.fortemi.index.record.v1",
|
|
11909
|
+
"aiwg.fortemi.index.record.v2"
|
|
11910
|
+
]
|
|
11911
|
+
},
|
|
11912
|
+
id: {
|
|
11913
|
+
type: "string",
|
|
11914
|
+
minLength: 1
|
|
11915
|
+
},
|
|
11916
|
+
type: {
|
|
11917
|
+
$ref: "#/$defs/recordType"
|
|
11918
|
+
},
|
|
11919
|
+
source: {
|
|
11920
|
+
type: "object",
|
|
11921
|
+
additionalProperties: false,
|
|
11922
|
+
required: ["path", "repo_relative_path", "locator"],
|
|
11923
|
+
properties: {
|
|
11924
|
+
path: {
|
|
11925
|
+
type: "string",
|
|
11926
|
+
minLength: 1
|
|
11927
|
+
},
|
|
11928
|
+
repo_relative_path: {
|
|
11929
|
+
type: "string",
|
|
11930
|
+
minLength: 1
|
|
11931
|
+
},
|
|
11932
|
+
locator: {
|
|
11933
|
+
type: "string",
|
|
11934
|
+
minLength: 1
|
|
11935
|
+
},
|
|
11936
|
+
origin: {
|
|
11937
|
+
type: "string",
|
|
11938
|
+
minLength: 1
|
|
11939
|
+
},
|
|
11940
|
+
generated: {
|
|
11941
|
+
type: "boolean"
|
|
11942
|
+
},
|
|
11943
|
+
checksum: {
|
|
11944
|
+
type: "string"
|
|
11945
|
+
},
|
|
11946
|
+
updated_at: {
|
|
11947
|
+
type: "string",
|
|
11948
|
+
format: "date-time"
|
|
11949
|
+
}
|
|
11950
|
+
}
|
|
11951
|
+
},
|
|
11952
|
+
title: {
|
|
11953
|
+
type: "string"
|
|
11954
|
+
},
|
|
11955
|
+
name: {
|
|
11956
|
+
type: "string"
|
|
11957
|
+
},
|
|
11958
|
+
summary: {
|
|
11959
|
+
type: "string"
|
|
11960
|
+
},
|
|
11961
|
+
text: {
|
|
11962
|
+
type: "string"
|
|
11963
|
+
},
|
|
11964
|
+
search: {
|
|
11965
|
+
type: "object",
|
|
11966
|
+
additionalProperties: false,
|
|
11967
|
+
required: [
|
|
11968
|
+
"title",
|
|
11969
|
+
"body",
|
|
11970
|
+
"triggers",
|
|
11971
|
+
"aliases",
|
|
11972
|
+
"tags",
|
|
11973
|
+
"frontmatter"
|
|
11974
|
+
],
|
|
11975
|
+
properties: {
|
|
11976
|
+
title: {
|
|
11977
|
+
type: "string"
|
|
11978
|
+
},
|
|
11979
|
+
name: {
|
|
11980
|
+
type: "string"
|
|
11981
|
+
},
|
|
11982
|
+
summary: {
|
|
11983
|
+
type: "string"
|
|
11984
|
+
},
|
|
11985
|
+
body: {
|
|
11986
|
+
type: "string"
|
|
11987
|
+
},
|
|
11988
|
+
triggers: {
|
|
11989
|
+
$ref: "#/$defs/stringArray"
|
|
11990
|
+
},
|
|
11991
|
+
aliases: {
|
|
11992
|
+
$ref: "#/$defs/stringArray"
|
|
11993
|
+
},
|
|
11994
|
+
capability: {
|
|
11995
|
+
type: "string"
|
|
11996
|
+
},
|
|
11997
|
+
tags: {
|
|
11998
|
+
$ref: "#/$defs/stringArray"
|
|
11999
|
+
},
|
|
12000
|
+
phase: {
|
|
12001
|
+
type: "string"
|
|
12002
|
+
},
|
|
12003
|
+
type: {
|
|
12004
|
+
type: "string"
|
|
12005
|
+
},
|
|
12006
|
+
frontmatter: {
|
|
12007
|
+
type: "object"
|
|
12008
|
+
}
|
|
12009
|
+
}
|
|
12010
|
+
},
|
|
12011
|
+
facets: {
|
|
12012
|
+
type: "object",
|
|
12013
|
+
additionalProperties: {
|
|
12014
|
+
$ref: "#/$defs/stringArray"
|
|
12015
|
+
}
|
|
12016
|
+
},
|
|
12017
|
+
tags: {
|
|
12018
|
+
$ref: "#/$defs/stringArray"
|
|
12019
|
+
},
|
|
12020
|
+
concepts: {
|
|
12021
|
+
$ref: "#/$defs/stringArray"
|
|
12022
|
+
},
|
|
12023
|
+
relationships: {
|
|
12024
|
+
type: "array",
|
|
12025
|
+
items: {
|
|
12026
|
+
type: "object",
|
|
12027
|
+
additionalProperties: false,
|
|
12028
|
+
required: ["type", "target_id"],
|
|
12029
|
+
properties: {
|
|
12030
|
+
type: {
|
|
12031
|
+
type: "string",
|
|
12032
|
+
minLength: 1
|
|
12033
|
+
},
|
|
12034
|
+
target_id: {
|
|
12035
|
+
type: "string",
|
|
12036
|
+
minLength: 1
|
|
12037
|
+
},
|
|
12038
|
+
source_path: {
|
|
12039
|
+
type: "string"
|
|
12040
|
+
},
|
|
12041
|
+
target_path: {
|
|
12042
|
+
type: "string"
|
|
12043
|
+
},
|
|
12044
|
+
direction: {
|
|
12045
|
+
enum: ["upstream", "downstream", "related"]
|
|
12046
|
+
},
|
|
12047
|
+
label: {
|
|
12048
|
+
type: "string"
|
|
12049
|
+
},
|
|
12050
|
+
confidence: {
|
|
12051
|
+
type: "number"
|
|
12052
|
+
},
|
|
12053
|
+
privacy: {
|
|
12054
|
+
$ref: "#/$defs/privacy"
|
|
12055
|
+
},
|
|
12056
|
+
metadata: {
|
|
12057
|
+
type: "object"
|
|
12058
|
+
}
|
|
12059
|
+
}
|
|
12060
|
+
}
|
|
12061
|
+
},
|
|
12062
|
+
provenance: {
|
|
12063
|
+
type: "array",
|
|
12064
|
+
minItems: 1,
|
|
12065
|
+
items: {
|
|
12066
|
+
type: "object",
|
|
12067
|
+
additionalProperties: false,
|
|
12068
|
+
required: ["field", "source", "path", "confidence", "privacy"],
|
|
12069
|
+
properties: {
|
|
12070
|
+
field: {
|
|
12071
|
+
type: "string",
|
|
12072
|
+
minLength: 1
|
|
12073
|
+
},
|
|
12074
|
+
source: {
|
|
12075
|
+
type: "string",
|
|
12076
|
+
minLength: 1
|
|
12077
|
+
},
|
|
12078
|
+
path: {
|
|
12079
|
+
type: "string",
|
|
12080
|
+
minLength: 1
|
|
12081
|
+
},
|
|
12082
|
+
confidence: {
|
|
12083
|
+
enum: ["source", "candidate", "reviewed", "rejected"]
|
|
12084
|
+
},
|
|
12085
|
+
privacy: {
|
|
12086
|
+
$ref: "#/$defs/privacy"
|
|
12087
|
+
}
|
|
12088
|
+
}
|
|
12089
|
+
}
|
|
12090
|
+
},
|
|
12091
|
+
privacy: {
|
|
12092
|
+
type: "object",
|
|
12093
|
+
additionalProperties: false,
|
|
12094
|
+
required: ["classification", "pii"],
|
|
12095
|
+
properties: {
|
|
12096
|
+
classification: {
|
|
12097
|
+
$ref: "#/$defs/privacy"
|
|
12098
|
+
},
|
|
12099
|
+
pii: {
|
|
12100
|
+
type: "boolean"
|
|
12101
|
+
},
|
|
12102
|
+
locality: {
|
|
12103
|
+
enum: ["project", "framework", "external"]
|
|
12104
|
+
}
|
|
12105
|
+
}
|
|
12106
|
+
},
|
|
12107
|
+
chunks: {
|
|
12108
|
+
type: "array",
|
|
12109
|
+
items: {
|
|
12110
|
+
type: "object",
|
|
12111
|
+
additionalProperties: false,
|
|
12112
|
+
properties: {
|
|
12113
|
+
id: {
|
|
12114
|
+
type: "string",
|
|
12115
|
+
minLength: 1
|
|
12116
|
+
},
|
|
12117
|
+
text: {
|
|
12118
|
+
type: "string"
|
|
12119
|
+
},
|
|
12120
|
+
body: {
|
|
12121
|
+
type: "string"
|
|
12122
|
+
},
|
|
12123
|
+
summary: {
|
|
12124
|
+
type: "string"
|
|
12125
|
+
},
|
|
12126
|
+
source_path: {
|
|
12127
|
+
type: "string"
|
|
12128
|
+
},
|
|
12129
|
+
metadata: {
|
|
12130
|
+
type: "object"
|
|
12131
|
+
},
|
|
12132
|
+
checksum: {
|
|
12133
|
+
type: "string"
|
|
12134
|
+
}
|
|
12135
|
+
}
|
|
12136
|
+
}
|
|
12137
|
+
},
|
|
12138
|
+
embeddings: {
|
|
12139
|
+
type: "array",
|
|
12140
|
+
items: {
|
|
12141
|
+
type: "object",
|
|
12142
|
+
additionalProperties: false,
|
|
12143
|
+
properties: {
|
|
12144
|
+
id: {
|
|
12145
|
+
type: "string"
|
|
12146
|
+
},
|
|
12147
|
+
model: {
|
|
12148
|
+
type: "string"
|
|
12149
|
+
},
|
|
12150
|
+
embedding: {
|
|
12151
|
+
$ref: "#/$defs/numberArray"
|
|
12152
|
+
},
|
|
12153
|
+
vector: {
|
|
12154
|
+
$ref: "#/$defs/numberArray"
|
|
12155
|
+
},
|
|
12156
|
+
granularity: {
|
|
12157
|
+
type: "string"
|
|
12158
|
+
},
|
|
12159
|
+
source_path: {
|
|
12160
|
+
type: "string"
|
|
12161
|
+
},
|
|
12162
|
+
metadata: {
|
|
12163
|
+
type: "object"
|
|
12164
|
+
},
|
|
12165
|
+
chunk_id: {
|
|
12166
|
+
type: "string"
|
|
12167
|
+
},
|
|
12168
|
+
vector_ref: {
|
|
12169
|
+
type: "string"
|
|
12170
|
+
},
|
|
12171
|
+
input_hash: {
|
|
12172
|
+
type: "string"
|
|
12173
|
+
}
|
|
12174
|
+
}
|
|
12175
|
+
}
|
|
12176
|
+
},
|
|
12177
|
+
compatibility: {
|
|
12178
|
+
type: "object"
|
|
12179
|
+
},
|
|
12180
|
+
skos_concepts: {
|
|
12181
|
+
type: "array",
|
|
12182
|
+
items: {
|
|
12183
|
+
type: "object",
|
|
12184
|
+
additionalProperties: false,
|
|
12185
|
+
required: ["id", "prefLabel"],
|
|
12186
|
+
properties: {
|
|
12187
|
+
id: {
|
|
12188
|
+
type: "string",
|
|
12189
|
+
minLength: 1
|
|
12190
|
+
},
|
|
12191
|
+
prefLabel: {
|
|
12192
|
+
type: "string",
|
|
12193
|
+
minLength: 1
|
|
12194
|
+
},
|
|
12195
|
+
definition: {
|
|
12196
|
+
type: "string"
|
|
12197
|
+
},
|
|
12198
|
+
scheme: {
|
|
12199
|
+
type: "string"
|
|
12200
|
+
},
|
|
12201
|
+
notation: {
|
|
12202
|
+
type: "string"
|
|
12203
|
+
},
|
|
12204
|
+
uri: {
|
|
12205
|
+
type: "string"
|
|
12206
|
+
},
|
|
12207
|
+
altLabels: {
|
|
12208
|
+
$ref: "#/$defs/stringArray"
|
|
12209
|
+
},
|
|
12210
|
+
metadata: {
|
|
12211
|
+
type: "object"
|
|
12212
|
+
}
|
|
12213
|
+
}
|
|
12214
|
+
}
|
|
12215
|
+
},
|
|
12216
|
+
skos_relations: {
|
|
12217
|
+
type: "array",
|
|
12218
|
+
items: {
|
|
12219
|
+
type: "object",
|
|
12220
|
+
additionalProperties: false,
|
|
12221
|
+
required: ["type", "source_id", "target_id"],
|
|
12222
|
+
properties: {
|
|
12223
|
+
type: {
|
|
12224
|
+
type: "string",
|
|
12225
|
+
minLength: 1
|
|
12226
|
+
},
|
|
12227
|
+
source_id: {
|
|
12228
|
+
type: "string",
|
|
12229
|
+
minLength: 1
|
|
12230
|
+
},
|
|
12231
|
+
target_id: {
|
|
12232
|
+
type: "string",
|
|
12233
|
+
minLength: 1
|
|
12234
|
+
},
|
|
12235
|
+
source_path: {
|
|
12236
|
+
type: "string"
|
|
12237
|
+
},
|
|
12238
|
+
metadata: {
|
|
12239
|
+
type: "object"
|
|
12240
|
+
}
|
|
12241
|
+
}
|
|
12242
|
+
}
|
|
12243
|
+
},
|
|
12244
|
+
provenance_events: {
|
|
12245
|
+
type: "array",
|
|
12246
|
+
items: {
|
|
12247
|
+
type: "object",
|
|
12248
|
+
additionalProperties: false,
|
|
12249
|
+
required: ["activity"],
|
|
12250
|
+
properties: {
|
|
12251
|
+
id: {
|
|
12252
|
+
type: "string"
|
|
12253
|
+
},
|
|
12254
|
+
activity: {
|
|
12255
|
+
type: "string",
|
|
12256
|
+
minLength: 1
|
|
12257
|
+
},
|
|
12258
|
+
agent: {
|
|
12259
|
+
type: "string"
|
|
12260
|
+
},
|
|
12261
|
+
started_at: {
|
|
12262
|
+
type: "string",
|
|
12263
|
+
format: "date-time"
|
|
12264
|
+
},
|
|
12265
|
+
ended_at: {
|
|
12266
|
+
type: "string",
|
|
12267
|
+
format: "date-time"
|
|
12268
|
+
},
|
|
12269
|
+
source: {
|
|
12270
|
+
type: "string"
|
|
12271
|
+
},
|
|
12272
|
+
path: {
|
|
12273
|
+
type: "string"
|
|
12274
|
+
},
|
|
12275
|
+
confidence: {
|
|
12276
|
+
enum: ["source", "candidate", "reviewed", "rejected"]
|
|
12277
|
+
},
|
|
12278
|
+
privacy: {
|
|
12279
|
+
$ref: "#/$defs/privacy"
|
|
12280
|
+
},
|
|
12281
|
+
attributes: {
|
|
12282
|
+
type: "object"
|
|
12283
|
+
}
|
|
12284
|
+
}
|
|
12285
|
+
}
|
|
12286
|
+
},
|
|
12287
|
+
updated_at: {
|
|
12288
|
+
type: "string",
|
|
12289
|
+
format: "date-time"
|
|
12290
|
+
}
|
|
12291
|
+
}
|
|
12292
|
+
}
|
|
12293
|
+
}
|
|
12294
|
+
};
|
|
12295
|
+
|
|
12296
|
+
// src/aiwg-index-schema.ts
|
|
12297
|
+
var PROJECTED_FIELDS = [
|
|
12298
|
+
"schema_version",
|
|
12299
|
+
"id",
|
|
12300
|
+
"type",
|
|
12301
|
+
"title",
|
|
12302
|
+
"text",
|
|
12303
|
+
"facets",
|
|
12304
|
+
"tags",
|
|
12305
|
+
"concepts",
|
|
12306
|
+
"privacy"
|
|
12307
|
+
];
|
|
12308
|
+
var ajvInstance2;
|
|
12309
|
+
var exportValidator;
|
|
12310
|
+
var projectedRecordValidator;
|
|
12311
|
+
function getAiwgFortemiIndexExportSchema() {
|
|
12312
|
+
return aiwg_fortemi_index_export_schema_default;
|
|
12313
|
+
}
|
|
12314
|
+
function getAjv2() {
|
|
12315
|
+
if (!ajvInstance2) {
|
|
12316
|
+
ajvInstance2 = new Ajv2020({
|
|
12317
|
+
allErrors: true,
|
|
12318
|
+
strict: false,
|
|
12319
|
+
validateFormats: false
|
|
12320
|
+
});
|
|
12321
|
+
ajvInstance2.addSchema(aiwg_fortemi_index_export_schema_default);
|
|
12322
|
+
}
|
|
12323
|
+
return ajvInstance2;
|
|
12324
|
+
}
|
|
12325
|
+
function formatErrors2(errors) {
|
|
12326
|
+
return (errors ?? []).map((error) => {
|
|
12327
|
+
const path = error.instancePath || "(root)";
|
|
12328
|
+
return `${path} ${error.message ?? "is invalid"}`;
|
|
12329
|
+
});
|
|
12330
|
+
}
|
|
12331
|
+
function getExportValidator() {
|
|
12332
|
+
exportValidator ??= getAjv2().getSchema(aiwg_fortemi_index_export_schema_default.$id) ?? getAjv2().compile(aiwg_fortemi_index_export_schema_default);
|
|
12333
|
+
return exportValidator;
|
|
12334
|
+
}
|
|
12335
|
+
function getProjectedRecordValidator() {
|
|
12336
|
+
if (!projectedRecordValidator) {
|
|
12337
|
+
const properties = Object.fromEntries(Object.keys(aiwg_fortemi_index_export_schema_default.$defs.record.properties).map((field) => [
|
|
12338
|
+
field,
|
|
12339
|
+
{ $ref: `${aiwg_fortemi_index_export_schema_default.$id}#/$defs/record/properties/${field}` }
|
|
12340
|
+
]));
|
|
12341
|
+
projectedRecordValidator = getAjv2().compile({
|
|
12342
|
+
type: "object",
|
|
12343
|
+
required: PROJECTED_FIELDS,
|
|
12344
|
+
properties,
|
|
12345
|
+
additionalProperties: true,
|
|
12346
|
+
allOf: [
|
|
12347
|
+
{
|
|
12348
|
+
if: {
|
|
12349
|
+
properties: {
|
|
12350
|
+
schema_version: { const: "aiwg.fortemi.index.record.v1" }
|
|
12351
|
+
}
|
|
12352
|
+
},
|
|
12353
|
+
then: { $ref: `${aiwg_fortemi_index_export_schema_default.$id}#/$defs/v1RecordCompatibility` }
|
|
12354
|
+
}
|
|
12355
|
+
]
|
|
12356
|
+
});
|
|
12357
|
+
}
|
|
12358
|
+
return projectedRecordValidator;
|
|
12359
|
+
}
|
|
12360
|
+
function validateAiwgFortemiIndexExportSchema(value) {
|
|
12361
|
+
const validate = getExportValidator();
|
|
12362
|
+
const schemaValue = value && typeof value === "object" && Array.isArray(value.items) ? {
|
|
12363
|
+
...value,
|
|
12364
|
+
items: value.items.map((item) => {
|
|
12365
|
+
if (!item || typeof item !== "object" || Array.isArray(item)) return item;
|
|
12366
|
+
const schemaRecord = { ...item };
|
|
12367
|
+
Reflect.deleteProperty(schemaRecord, "binary_sources");
|
|
12368
|
+
return schemaRecord;
|
|
12369
|
+
})
|
|
12370
|
+
} : value;
|
|
12371
|
+
const valid = validate(schemaValue);
|
|
12372
|
+
return { valid, errors: formatErrors2(validate.errors) };
|
|
12373
|
+
}
|
|
12374
|
+
function validateAiwgFortemiProjectedRecordSchema(value) {
|
|
12375
|
+
const validate = getProjectedRecordValidator();
|
|
12376
|
+
const valid = validate(value);
|
|
12377
|
+
return { valid, errors: formatErrors2(validate.errors) };
|
|
12378
|
+
}
|
|
12379
|
+
|
|
9671
12380
|
// src/index.ts
|
|
9672
|
-
var VERSION = "2026.7.
|
|
12381
|
+
var VERSION = "2026.7.5";
|
|
9673
12382
|
|
|
9674
|
-
export { AIWG_SCAN_REQUIRED_FIELDS, ArchiveManager, AttachmentsRepository, CURRENT_MIGRATION_HEAD, CURRENT_SHARD_VERSION, CapabilityManager, CaptureKnowledgeInputSchema, CollectionsRepository, CommunitiesRepository, DB_SNAPSHOT_SCHEMA_VERSION, DbSnapshotVersionError, EMBED_REQUEST_KIND, EMBED_RESPONSE_KIND, EmbeddingSetsRepository, FallbackRouter, FortemiToolManifest, GetNoteInputSchema, GraphRepository, JOB_CAPABILITIES, JOB_PRIORITIES, JobQueueWorker, LOCAL_ENDPOINTS, LinksRepository, ListNotesInputSchema, ManageArchiveInputSchema, ManageAttachmentsInputSchema, ManageCapabilitiesInputSchema, ManageCollectionsInputSchema, ManageLinksInputSchema, ManageNoteInputSchema, ManageTagsInputSchema, MemoryBlobStore, MigrationRunner, NotesRepository, OpenAICompatibleProvider, PGliteStorageBackend, PGliteStorageBackendFactory, PGliteWorkerClient, PGliteWorkerStorageBackend, PGliteWorkerStorageBackendFactory, ProvenanceRepository, ProviderRegistry, SHARD_FORMAT, SUPPORTED_PGLITE_VERSION, SearchInputSchema, SearchRepository, SkosRepository, TagsRepository, TransactionProxy, TypedEventBus, VERSION, aiRevisionHandler, aiwgFortemiIndexToCommunityGraph, allMigrations, appendPluginScript, assertAiwgFortemiChunkManifest, assertAiwgFortemiChunkPart, assertAiwgFortemiIndexExport, assertAiwgStaticEmbeddingSet, buildAiwgChunkedIndex, buildAiwgStaticEmbeddingSet, buildNoteConditions, buildPluginCsp, captureKnowledge, chunkText, classifyError, classifyModel, clearPrefetchedShard, collectionFromShard, collectionToShard, computeHash, computeSri, conceptTaggingHandler, cosineSimilarity, createAiwgFetchChunkLoader, createAiwgFetchDetailLoader, createAiwgIndexController, createAiwgReviewDecisionExport, createBlobStore, createCosineSemanticProvider, createCspReportHandler, createFortemi, createLegacyProvider, createPGliteBackend, createPGliteInstance, createRemoteBackend, createRoutes, createShardBackend, createWorkerEmbedFunction, defaultStorageBackendFactory, detectCommunities, detectGpuCapabilities, detectInferenceCapabilities, discoverLocalProviders, dumpDbSnapshot, embeddingFromShard, embeddingGenerationHandler, embeddingSetFromShard, embeddingSetMemberToShard, embeddingSetToShard, embeddingToShard, enqueueFullWorkflow, enqueueJob, enqueueNoteCreationJobs, estimateModelFit, estimateVramMB, estimateVramTier, exportShard, fetchPluginScript, filterAiwgRecordsByPrivacy, findAiwgStaticDuplicatePairs, fortemiManifest, fromPrefetched, generateId, getAiwgFortemiFacets, getEmbedFunction, getFortemiBridge, getFortemiSecretStore, getJobQueueStatus, getLlmFunction, getNote, getPrefetchedSha256, handleEmbedRequests, hasFortemiSecureSecrets, importShard, isPluginScriptAllowed, isShardPrefetched, linkFromShard, linkToShard, linkingHandler, listNotes, manageArchive, manageAttachments, manageCapabilities, manageCollections, manageLinks, manageNote, manageTags, matchRoute, noteFromShard, noteSkosTagToShard, noteToShard, openShard, packTarGz, parseCspReport, prefetchShard, provenanceEdgeToShard, queryAiwgFortemiIndex, queryAiwgHybridIndex, queryAiwgSemanticIndex, registerLlmCapability, registerSemanticCapability, registerSemanticCapabilityWorker, registerServiceWorker, restoreDbSnapshot, searchTool, selectBackend, selectLlmModel, setEmbedFunction, setLlmFunction, sha256Hex, skosConceptToShard, skosRelationToShard, skosSchemeToShard, suggestTags,
|
|
12383
|
+
export { AIWG_SCAN_REQUIRED_FIELDS, ArchiveManager, AttachmentsRepository, CURRENT_MIGRATION_HEAD, CURRENT_SHARD_VERSION, CapabilityManager, CaptureKnowledgeInputSchema, CollectionsRepository, CommunitiesRepository, DB_SNAPSHOT_SCHEMA_VERSION, DbSnapshotVersionError, EMBED_REQUEST_KIND, EMBED_RESPONSE_KIND, EmbeddingSetsRepository, FORTEMI_COMPATIBILITY_PATH, FORTEMI_COMPATIBILITY_STATES, FORTEMI_REQUIRED_COMPATIBILITY_CAPABILITIES, FallbackRouter, FortemiToolManifest, GetNoteInputSchema, GraphRepository, JOB_CAPABILITIES, JOB_PRIORITIES, JobQueueWorker, LOCAL_ENDPOINTS, LinksRepository, ListNotesInputSchema, ManageArchiveInputSchema, ManageAttachmentsInputSchema, ManageCapabilitiesInputSchema, ManageCollectionsInputSchema, ManageLinksInputSchema, ManageNoteInputSchema, ManageTagsInputSchema, MemoryBlobStore, MigrationRunner, NotesRepository, OpenAICompatibleProvider, PGliteStorageBackend, PGliteStorageBackendFactory, PGliteWorkerClient, PGliteWorkerStorageBackend, PGliteWorkerStorageBackendFactory, ProvenanceRepository, ProviderRegistry, SHARD_FORMAT, SUPPORTED_PGLITE_VERSION, SearchInputSchema, SearchRepository, SkosRepository, TagsRepository, TransactionProxy, TypedEventBus, VERSION, aiRevisionHandler, aiwgFortemiIndexToCommunityGraph, allMigrations, appendPluginScript, assertAiwgFortemiChunkManifest, assertAiwgFortemiChunkPart, assertAiwgFortemiIndexExport, assertAiwgStaticEmbeddingSet, assertShardComponentRecord, buildAiwgChunkedIndex, buildAiwgStaticEmbeddingSet, buildNoteConditions, buildPluginCsp, captureKnowledge, chunkText, classifyError, classifyModel, clearPrefetchedShard, collectionFromShard, collectionToShard, computeBlobHash, computeHash, computeSri, conceptTaggingHandler, cosineSimilarity, createAiwgFetchChunkLoader, createAiwgFetchDetailLoader, createAiwgIndexController, createAiwgReviewDecisionExport, createBlobStore, createCosineSemanticProvider, createCspReportHandler, createFortemi, createLegacyProvider, createPGliteBackend, createPGliteInstance, createRemoteBackend, createRoutes, createShardBackend, createWorkerEmbedFunction, defaultStorageBackendFactory, detectCommunities, detectGpuCapabilities, detectInferenceCapabilities, discoverLocalProviders, dumpDbSnapshot, embeddingConfigToShard, embeddingFromShard, embeddingGenerationHandler, embeddingSetFromShard, embeddingSetMemberToShard, embeddingSetToShard, embeddingToShard, enqueueFullWorkflow, enqueueJob, enqueueNoteCreationJobs, estimateModelFit, estimateVramMB, estimateVramTier, exportShard, fetchAndValidateFortemiCompatibility, fetchPluginScript, filterAiwgRecordsByPrivacy, findAiwgStaticDuplicatePairs, formatFortemiCompatibilitySummary, fortemiCompatibilityUrl, fortemiManifest, fromPrefetched, generateId, getAiwgFortemiFacets, getAiwgFortemiIndexExportSchema, getEmbedFunction, getFortemiBridge, getFortemiSecretStore, getJobQueueStatus, getKnowledgeShardSchema, getLlmFunction, getNote, getPrefetchedSha256, handleEmbedRequests, hasFortemiSecureSecrets, importShard, isPluginScriptAllowed, isShardPrefetched, linkFromShard, linkToShard, linkingHandler, listNotes, manageArchive, manageAttachments, manageCapabilities, manageCollections, manageLinks, manageNote, manageTags, matchRoute, noteFromShard, noteSkosTagToShard, noteToShard, openShard, packTarGz, parseCspReport, prefetchShard, provenanceEdgeToShard, queryAiwgFortemiIndex, queryAiwgHybridIndex, queryAiwgSemanticIndex, registerLlmCapability, registerSemanticCapability, registerSemanticCapabilityWorker, registerServiceWorker, restoreDbSnapshot, searchTool, selectBackend, selectLlmModel, setEmbedFunction, setLlmFunction, sha256Hex, skosConceptToShard, skosRelationToShard, skosSchemeToShard, suggestTags, tagsToShard, templateToShard, titleGenerationHandler, unpackTarGz, unregisterLlmCapability, unregisterSemanticCapability, urlLinkToShard, validateAiwgFortemiChunkManifest, validateAiwgFortemiChunkPart, validateAiwgFortemiIndexExport, validateAiwgFortemiIndexExportSchema, validateAiwgFortemiProjectedRecordSchema, validateAiwgStaticEmbeddingSet, validateChecksums, validateFortemiCompatibilityResponse, validateShardArchive, validateShardComponentRecord, validateShardManifest, verifyDbSnapshotMeta, verifySri };
|
|
9675
12384
|
//# sourceMappingURL=index.js.map
|
|
9676
12385
|
//# sourceMappingURL=index.js.map
|