@fortemi/core 2026.7.3 → 2026.7.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/dist/aiwg-index-C2G7iy-b.d.ts +826 -0
- package/dist/aiwg-index.d.ts +1 -478
- package/dist/aiwg-index.js +992 -80
- package/dist/aiwg-index.js.map +1 -1
- package/dist/index.d.ts +223 -328
- package/dist/index.js +2763 -222
- package/dist/index.js.map +1 -1
- package/package.json +15 -3
- package/tools/verify-fortemi-compatibility.mjs +82 -0
package/dist/index.js
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import { v7 } from 'uuid';
|
|
2
|
-
import { PGlite } from '@electric-sql/pglite';
|
|
3
|
-
import { vector } from '@electric-sql/pglite/vector';
|
|
4
2
|
import { sha256 } from '@noble/hashes/sha256';
|
|
3
|
+
import { blake3 } from '@noble/hashes/blake3';
|
|
5
4
|
import { bytesToHex } from '@noble/hashes/utils';
|
|
6
5
|
import { z } from 'zod';
|
|
7
6
|
import { gzipSync, gunzipSync } from 'fflate';
|
|
7
|
+
import Ajv2020 from 'ajv/dist/2020.js';
|
|
8
8
|
|
|
9
9
|
// src/uuid.ts
|
|
10
10
|
function generateId() {
|
|
@@ -125,6 +125,8 @@ var TypedEventBus = class {
|
|
|
125
125
|
this.wildcardListeners.clear();
|
|
126
126
|
}
|
|
127
127
|
};
|
|
128
|
+
|
|
129
|
+
// src/db.ts
|
|
128
130
|
function getDataDir(persistence, archiveName) {
|
|
129
131
|
switch (persistence) {
|
|
130
132
|
case "opfs":
|
|
@@ -137,6 +139,10 @@ function getDataDir(persistence, archiveName) {
|
|
|
137
139
|
}
|
|
138
140
|
async function createPGliteInstance(persistence, archiveName = "default", options = {}) {
|
|
139
141
|
const dataDir = getDataDir(persistence, archiveName);
|
|
142
|
+
const [{ PGlite }, { vector }] = await Promise.all([
|
|
143
|
+
import('@electric-sql/pglite'),
|
|
144
|
+
import('@electric-sql/pglite/vector')
|
|
145
|
+
]);
|
|
140
146
|
const pgliteOptions = {
|
|
141
147
|
database: "postgres",
|
|
142
148
|
// PGlite 0.4.x breaking change: explicit required
|
|
@@ -609,6 +615,134 @@ var migration0010 = {
|
|
|
609
615
|
`
|
|
610
616
|
};
|
|
611
617
|
|
|
618
|
+
// src/migrations/0011_embedding_member_metadata.ts
|
|
619
|
+
var migration0011 = {
|
|
620
|
+
version: 11,
|
|
621
|
+
name: "0011_embedding_member_metadata",
|
|
622
|
+
sql: `
|
|
623
|
+
ALTER TABLE embedding_set_member
|
|
624
|
+
ALTER COLUMN embedding_id DROP NOT NULL;
|
|
625
|
+
|
|
626
|
+
ALTER TABLE embedding_set_member
|
|
627
|
+
ADD COLUMN IF NOT EXISTS membership_type TEXT NOT NULL DEFAULT 'materialized';
|
|
628
|
+
|
|
629
|
+
ALTER TABLE embedding_set_member
|
|
630
|
+
ADD COLUMN IF NOT EXISTS added_at TIMESTAMPTZ NOT NULL DEFAULT now();
|
|
631
|
+
|
|
632
|
+
ALTER TABLE embedding_set_member
|
|
633
|
+
ADD COLUMN IF NOT EXISTS added_by TEXT;
|
|
634
|
+
`
|
|
635
|
+
};
|
|
636
|
+
|
|
637
|
+
// src/migrations/0012_embedding_configs.ts
|
|
638
|
+
var migration0012 = {
|
|
639
|
+
version: 12,
|
|
640
|
+
name: "0012_embedding_configs",
|
|
641
|
+
sql: `
|
|
642
|
+
CREATE TABLE IF NOT EXISTS embedding_config (
|
|
643
|
+
id TEXT PRIMARY KEY,
|
|
644
|
+
name TEXT NOT NULL,
|
|
645
|
+
description TEXT,
|
|
646
|
+
model TEXT NOT NULL,
|
|
647
|
+
dimension INTEGER NOT NULL,
|
|
648
|
+
chunk_size INTEGER NOT NULL,
|
|
649
|
+
chunk_overlap INTEGER NOT NULL,
|
|
650
|
+
is_default BOOLEAN NOT NULL DEFAULT false
|
|
651
|
+
);
|
|
652
|
+
|
|
653
|
+
CREATE INDEX IF NOT EXISTS idx_embedding_config_default ON embedding_config(is_default);
|
|
654
|
+
`
|
|
655
|
+
};
|
|
656
|
+
|
|
657
|
+
// src/migrations/0013_templates.ts
|
|
658
|
+
var migration0013 = {
|
|
659
|
+
version: 13,
|
|
660
|
+
name: "0013_templates",
|
|
661
|
+
sql: `
|
|
662
|
+
CREATE TABLE IF NOT EXISTS template (
|
|
663
|
+
id TEXT PRIMARY KEY,
|
|
664
|
+
name TEXT NOT NULL,
|
|
665
|
+
description TEXT,
|
|
666
|
+
content TEXT NOT NULL,
|
|
667
|
+
format TEXT NOT NULL DEFAULT 'markdown',
|
|
668
|
+
default_tags JSONB NOT NULL DEFAULT '[]',
|
|
669
|
+
collection_id TEXT,
|
|
670
|
+
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
|
671
|
+
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
|
672
|
+
);
|
|
673
|
+
|
|
674
|
+
CREATE INDEX IF NOT EXISTS idx_template_collection ON template(collection_id);
|
|
675
|
+
`
|
|
676
|
+
};
|
|
677
|
+
|
|
678
|
+
// src/migrations/0014_url_links.ts
|
|
679
|
+
var migration0014 = {
|
|
680
|
+
version: 14,
|
|
681
|
+
name: "0014_url_links",
|
|
682
|
+
sql: `
|
|
683
|
+
CREATE TABLE IF NOT EXISTS link_url_target (
|
|
684
|
+
id TEXT PRIMARY KEY,
|
|
685
|
+
source_note_id TEXT NOT NULL,
|
|
686
|
+
to_url TEXT NOT NULL,
|
|
687
|
+
link_type TEXT NOT NULL DEFAULT 'reference',
|
|
688
|
+
confidence REAL,
|
|
689
|
+
metadata_json JSONB,
|
|
690
|
+
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
|
691
|
+
updated_at TIMESTAMPTZ,
|
|
692
|
+
deleted_at TIMESTAMPTZ
|
|
693
|
+
);
|
|
694
|
+
|
|
695
|
+
CREATE INDEX IF NOT EXISTS idx_link_url_source ON link_url_target(source_note_id);
|
|
696
|
+
`
|
|
697
|
+
};
|
|
698
|
+
|
|
699
|
+
// src/migrations/0015_embedding_set_server_metadata.ts
|
|
700
|
+
var migration0015 = {
|
|
701
|
+
version: 15,
|
|
702
|
+
name: "0015_embedding_set_server_metadata",
|
|
703
|
+
sql: `
|
|
704
|
+
ALTER TABLE embedding_set
|
|
705
|
+
ADD COLUMN IF NOT EXISTS slug TEXT;
|
|
706
|
+
|
|
707
|
+
ALTER TABLE embedding_set
|
|
708
|
+
ADD COLUMN IF NOT EXISTS description TEXT;
|
|
709
|
+
|
|
710
|
+
ALTER TABLE embedding_set
|
|
711
|
+
ADD COLUMN IF NOT EXISTS document_count INTEGER;
|
|
712
|
+
|
|
713
|
+
ALTER TABLE embedding_set
|
|
714
|
+
ADD COLUMN IF NOT EXISTS embedding_count INTEGER;
|
|
715
|
+
|
|
716
|
+
ALTER TABLE embedding_set
|
|
717
|
+
ADD COLUMN IF NOT EXISTS is_system BOOLEAN NOT NULL DEFAULT FALSE;
|
|
718
|
+
|
|
719
|
+
ALTER TABLE embedding_set
|
|
720
|
+
ADD COLUMN IF NOT EXISTS keywords_json JSONB;
|
|
721
|
+
`
|
|
722
|
+
};
|
|
723
|
+
|
|
724
|
+
// src/migrations/0016_embedding_server_metadata.ts
|
|
725
|
+
var migration0016 = {
|
|
726
|
+
version: 16,
|
|
727
|
+
name: "0016_embedding_server_metadata",
|
|
728
|
+
sql: `
|
|
729
|
+
ALTER TABLE embedding
|
|
730
|
+
ADD COLUMN IF NOT EXISTS chunk_index INTEGER NOT NULL DEFAULT 0;
|
|
731
|
+
|
|
732
|
+
ALTER TABLE embedding
|
|
733
|
+
ADD COLUMN IF NOT EXISTS text TEXT NOT NULL DEFAULT '';
|
|
734
|
+
|
|
735
|
+
ALTER TABLE embedding
|
|
736
|
+
ADD COLUMN IF NOT EXISTS model TEXT;
|
|
737
|
+
|
|
738
|
+
ALTER TABLE embedding
|
|
739
|
+
DROP CONSTRAINT IF EXISTS embedding_note_id_embedding_set_id_key;
|
|
740
|
+
|
|
741
|
+
CREATE UNIQUE INDEX IF NOT EXISTS idx_embedding_note_set_chunk
|
|
742
|
+
ON embedding(note_id, embedding_set_id, chunk_index);
|
|
743
|
+
`
|
|
744
|
+
};
|
|
745
|
+
|
|
612
746
|
// src/migrations/index.ts
|
|
613
747
|
var allMigrations = [
|
|
614
748
|
migration0001,
|
|
@@ -620,7 +754,13 @@ var allMigrations = [
|
|
|
620
754
|
migration0007,
|
|
621
755
|
migration0008,
|
|
622
756
|
migration0009,
|
|
623
|
-
migration0010
|
|
757
|
+
migration0010,
|
|
758
|
+
migration0011,
|
|
759
|
+
migration0012,
|
|
760
|
+
migration0013,
|
|
761
|
+
migration0014,
|
|
762
|
+
migration0015,
|
|
763
|
+
migration0016
|
|
624
764
|
];
|
|
625
765
|
|
|
626
766
|
// src/data-archive.ts
|
|
@@ -730,6 +870,9 @@ function computeHash(data) {
|
|
|
730
870
|
const digest = sha256(data);
|
|
731
871
|
return `sha256:${bytesToHex(digest)}`;
|
|
732
872
|
}
|
|
873
|
+
function computeBlobHash(data) {
|
|
874
|
+
return `blake3:${bytesToHex(blake3(data))}`;
|
|
875
|
+
}
|
|
733
876
|
|
|
734
877
|
// src/repositories/notes-repository.ts
|
|
735
878
|
var NotesRepository = class {
|
|
@@ -1288,11 +1431,11 @@ var EmbeddingSetsRepository = class {
|
|
|
1288
1431
|
[input.note_id, input.embedding_set_id]
|
|
1289
1432
|
);
|
|
1290
1433
|
const embeddingId = input.id ?? generateId();
|
|
1291
|
-
const
|
|
1434
|
+
const vector = `[${input.vector.join(",")}]`;
|
|
1292
1435
|
await this.db.query(
|
|
1293
1436
|
`INSERT INTO embedding (id, note_id, embedding_set_id, vector)
|
|
1294
1437
|
VALUES ($1, $2, $3, $4::vector)`,
|
|
1295
|
-
[embeddingId, input.note_id, input.embedding_set_id,
|
|
1438
|
+
[embeddingId, input.note_id, input.embedding_set_id, vector]
|
|
1296
1439
|
);
|
|
1297
1440
|
await this.db.query(
|
|
1298
1441
|
`INSERT INTO embedding_set_member (embedding_set_id, note_id, embedding_id)
|
|
@@ -1454,7 +1597,7 @@ var EmbeddingSetsRepository = class {
|
|
|
1454
1597
|
if (criteria.conceptIds && criteria.conceptIds.length > 0) {
|
|
1455
1598
|
throw new Error("Unsupported virtual embedding-set criteria field: conceptIds");
|
|
1456
1599
|
}
|
|
1457
|
-
const conditions = ["e.embedding_set_id = $1"];
|
|
1600
|
+
const conditions = ["e.embedding_set_id = $1", "n.deleted_at IS NULL"];
|
|
1458
1601
|
const params = [source.baseSetId];
|
|
1459
1602
|
let idx = 2;
|
|
1460
1603
|
if (criteria.noteIds?.length) {
|
|
@@ -1657,7 +1800,7 @@ var ATTACHMENT_TEXT_JOIN2 = `
|
|
|
1657
1800
|
WHERE deleted_at IS NULL
|
|
1658
1801
|
GROUP BY note_id
|
|
1659
1802
|
) ax ON ax.note_id = n.id`;
|
|
1660
|
-
var COMBINED_TEXT_SQL = `(coalesce(c.content, '') || ' ' || coalesce(ax.extracted_text, ''))`;
|
|
1803
|
+
var COMBINED_TEXT_SQL = `trim(both from (coalesce(c.content, '') || ' ' || coalesce(ax.extracted_text, '')))`;
|
|
1661
1804
|
var COMBINED_TEXT_VECTOR_SQL2 = `to_tsvector('english', ${COMBINED_TEXT_SQL})`;
|
|
1662
1805
|
var SearchRepository = class {
|
|
1663
1806
|
constructor(db, semanticAvailable = false) {
|
|
@@ -2285,11 +2428,25 @@ function linkToBackend(link) {
|
|
|
2285
2428
|
createdAt: toIso(link.created_at)
|
|
2286
2429
|
};
|
|
2287
2430
|
}
|
|
2431
|
+
function urlLinkToBackend(link) {
|
|
2432
|
+
const metadata = typeof link.metadata_json === "string" ? JSON.parse(link.metadata_json) : link.metadata_json ?? void 0;
|
|
2433
|
+
return {
|
|
2434
|
+
id: link.id,
|
|
2435
|
+
fromNoteId: link.source_note_id,
|
|
2436
|
+
toNoteId: null,
|
|
2437
|
+
toUrl: link.to_url,
|
|
2438
|
+
kind: link.link_type,
|
|
2439
|
+
score: link.confidence,
|
|
2440
|
+
createdAt: toIso(link.created_at),
|
|
2441
|
+
...metadata ? { metadata } : {}
|
|
2442
|
+
};
|
|
2443
|
+
}
|
|
2288
2444
|
function shardLinkToBackend(link) {
|
|
2289
2445
|
return {
|
|
2290
2446
|
id: link.id,
|
|
2291
2447
|
fromNoteId: link.from_note_id,
|
|
2292
2448
|
toNoteId: link.to_note_id,
|
|
2449
|
+
toUrl: link.to_url,
|
|
2293
2450
|
kind: link.kind,
|
|
2294
2451
|
score: link.score,
|
|
2295
2452
|
createdAt: link.created_at,
|
|
@@ -2335,7 +2492,8 @@ function remoteLinkToBackend(link) {
|
|
|
2335
2492
|
return {
|
|
2336
2493
|
id: link.id,
|
|
2337
2494
|
fromNoteId: link.fromNoteId ?? link.from_note_id ?? link.source_note_id ?? "",
|
|
2338
|
-
toNoteId: link.toNoteId ?? link.to_note_id ?? link.target_note_id ??
|
|
2495
|
+
toNoteId: link.toNoteId ?? link.to_note_id ?? link.target_note_id ?? null,
|
|
2496
|
+
toUrl: link.toUrl ?? link.to_url ?? null,
|
|
2339
2497
|
kind: link.kind ?? link.link_type ?? "",
|
|
2340
2498
|
score: link.score ?? link.confidence ?? null,
|
|
2341
2499
|
createdAt: link.createdAt ?? link.created_at ?? "",
|
|
@@ -2357,7 +2515,15 @@ function createPGliteBackend(db, options = {}) {
|
|
|
2357
2515
|
const links = new LinksRepository(db);
|
|
2358
2516
|
async function linksOf(id) {
|
|
2359
2517
|
const result = await links.listForNote(id);
|
|
2360
|
-
|
|
2518
|
+
const urlLinks = await db.query(
|
|
2519
|
+
`SELECT * FROM link_url_target WHERE source_note_id = $1 AND deleted_at IS NULL ORDER BY created_at`,
|
|
2520
|
+
[id]
|
|
2521
|
+
);
|
|
2522
|
+
return [
|
|
2523
|
+
...result.outbound.map(linkToBackend),
|
|
2524
|
+
...result.inbound.map(linkToBackend),
|
|
2525
|
+
...urlLinks.rows.map(urlLinkToBackend)
|
|
2526
|
+
];
|
|
2361
2527
|
}
|
|
2362
2528
|
async function conceptsOf(id) {
|
|
2363
2529
|
const result = await db.query(
|
|
@@ -2988,6 +3154,20 @@ var MigrationRunner = class {
|
|
|
2988
3154
|
|
|
2989
3155
|
// src/archive-manager.ts
|
|
2990
3156
|
var ArchiveManager = class {
|
|
3157
|
+
/**
|
|
3158
|
+
* Persistence is a PLUGGABLE, opt-in backend (issue #261). Pass a
|
|
3159
|
+
* `StorageBackendFactory` to run against any `StorageBackend`
|
|
3160
|
+
* implementation; PGlite is merely the built-in default, selected only when
|
|
3161
|
+
* the convenience `PersistenceMode` string form is used. Because
|
|
3162
|
+
* `defaultStorageBackendFactory` reaches PGlite through the lazily-imported
|
|
3163
|
+
* `createPGliteInstance` (see `db.ts`), a consumer that supplies its own
|
|
3164
|
+
* factory — or never opens an archive at all — never pulls the PGlite WASM
|
|
3165
|
+
* engine into its bundle.
|
|
3166
|
+
*
|
|
3167
|
+
* @param persistenceOrFactory `PersistenceMode` string (uses the built-in
|
|
3168
|
+
* PGlite backend) OR a custom `StorageBackendFactory`.
|
|
3169
|
+
* @param persistenceOverride persistence hint passed to a custom factory.
|
|
3170
|
+
*/
|
|
2991
3171
|
constructor(persistenceOrFactory, events, persistenceOverride) {
|
|
2992
3172
|
this.events = events;
|
|
2993
3173
|
if (typeof persistenceOrFactory === "string") {
|
|
@@ -3096,6 +3276,205 @@ function createFortemi(config) {
|
|
|
3096
3276
|
};
|
|
3097
3277
|
}
|
|
3098
3278
|
|
|
3279
|
+
// src/server-compatibility.ts
|
|
3280
|
+
var FORTEMI_COMPATIBILITY_PATH = "/api/v1/system/compatibility";
|
|
3281
|
+
var FORTEMI_COMPATIBILITY_STATES = [
|
|
3282
|
+
"available",
|
|
3283
|
+
"degraded",
|
|
3284
|
+
"preview",
|
|
3285
|
+
"unavailable",
|
|
3286
|
+
"unknown"
|
|
3287
|
+
];
|
|
3288
|
+
var FORTEMI_REQUIRED_COMPATIBILITY_CAPABILITIES = [
|
|
3289
|
+
"core_notes",
|
|
3290
|
+
"search",
|
|
3291
|
+
"jobs",
|
|
3292
|
+
"realtime_activity",
|
|
3293
|
+
"hosted_auth",
|
|
3294
|
+
"premium_components",
|
|
3295
|
+
"backoffice_api",
|
|
3296
|
+
"audit_posture",
|
|
3297
|
+
"quota_status",
|
|
3298
|
+
"kms_status",
|
|
3299
|
+
"mcp_scope_gate"
|
|
3300
|
+
];
|
|
3301
|
+
function isRecord(value) {
|
|
3302
|
+
return !!value && typeof value === "object" && !Array.isArray(value);
|
|
3303
|
+
}
|
|
3304
|
+
function isSupportedState(value) {
|
|
3305
|
+
return typeof value === "string" && FORTEMI_COMPATIBILITY_STATES.includes(value);
|
|
3306
|
+
}
|
|
3307
|
+
function requireRecord(parent, key, errors) {
|
|
3308
|
+
const value = parent[key];
|
|
3309
|
+
if (!isRecord(value)) {
|
|
3310
|
+
errors.push(`${key} must be an object`);
|
|
3311
|
+
return null;
|
|
3312
|
+
}
|
|
3313
|
+
return value;
|
|
3314
|
+
}
|
|
3315
|
+
function requireString(parent, key, path, errors) {
|
|
3316
|
+
const value = parent[key];
|
|
3317
|
+
if (typeof value !== "string" || value.length === 0) {
|
|
3318
|
+
errors.push(`${path}.${key} must be a non-empty string`);
|
|
3319
|
+
return null;
|
|
3320
|
+
}
|
|
3321
|
+
return value;
|
|
3322
|
+
}
|
|
3323
|
+
function requireBoolean(parent, key, path, errors) {
|
|
3324
|
+
const value = parent[key];
|
|
3325
|
+
if (typeof value !== "boolean") {
|
|
3326
|
+
errors.push(`${path}.${key} must be a boolean`);
|
|
3327
|
+
return null;
|
|
3328
|
+
}
|
|
3329
|
+
return value;
|
|
3330
|
+
}
|
|
3331
|
+
function fortemiCompatibilityUrl(baseUrl = "http://localhost:3000") {
|
|
3332
|
+
const trimmed = baseUrl.trim().replace(/\/+$/, "");
|
|
3333
|
+
if (trimmed.endsWith(FORTEMI_COMPATIBILITY_PATH)) return trimmed;
|
|
3334
|
+
return `${trimmed}${FORTEMI_COMPATIBILITY_PATH}`;
|
|
3335
|
+
}
|
|
3336
|
+
function validateFortemiCompatibilityResponse(raw) {
|
|
3337
|
+
const errors = [];
|
|
3338
|
+
const warnings = [];
|
|
3339
|
+
if (!isRecord(raw)) {
|
|
3340
|
+
return {
|
|
3341
|
+
ok: false,
|
|
3342
|
+
errors: ["response must be a JSON object"],
|
|
3343
|
+
warnings
|
|
3344
|
+
};
|
|
3345
|
+
}
|
|
3346
|
+
if (raw.schema_version !== 1) {
|
|
3347
|
+
errors.push("schema_version must be 1");
|
|
3348
|
+
}
|
|
3349
|
+
requireString(raw, "contract_revision", "response", errors);
|
|
3350
|
+
const api = requireRecord(raw, "api", errors);
|
|
3351
|
+
if (api) {
|
|
3352
|
+
const name = requireString(api, "name", "api", errors);
|
|
3353
|
+
if (name && name !== "fortemi") {
|
|
3354
|
+
errors.push("api.name must be fortemi");
|
|
3355
|
+
}
|
|
3356
|
+
requireString(api, "version", "api", errors);
|
|
3357
|
+
requireString(api, "minimum_hotm_enterprise_client", "api", errors);
|
|
3358
|
+
requireBoolean(api, "git_sha_present", "api", errors);
|
|
3359
|
+
requireBoolean(api, "build_date_present", "api", errors);
|
|
3360
|
+
}
|
|
3361
|
+
const deployment = requireRecord(raw, "deployment", errors);
|
|
3362
|
+
if (deployment) {
|
|
3363
|
+
requireString(deployment, "mode", "deployment", errors);
|
|
3364
|
+
requireString(deployment, "edition", "deployment", errors);
|
|
3365
|
+
requireBoolean(deployment, "hosted_multi_tenant_ready", "deployment", errors);
|
|
3366
|
+
}
|
|
3367
|
+
const auth = requireRecord(raw, "auth", errors);
|
|
3368
|
+
if (auth) {
|
|
3369
|
+
requireBoolean(auth, "required", "auth", errors);
|
|
3370
|
+
requireString(auth, "mode", "auth", errors);
|
|
3371
|
+
requireBoolean(auth, "oauth_issuer_configured", "auth", errors);
|
|
3372
|
+
requireBoolean(auth, "tenant_context_available", "auth", errors);
|
|
3373
|
+
}
|
|
3374
|
+
const capabilities = requireRecord(raw, "capabilities", errors);
|
|
3375
|
+
if (capabilities) {
|
|
3376
|
+
for (const key of FORTEMI_REQUIRED_COMPATIBILITY_CAPABILITIES) {
|
|
3377
|
+
if (!isRecord(capabilities[key])) {
|
|
3378
|
+
errors.push(`capabilities.${key} must be present as an object`);
|
|
3379
|
+
}
|
|
3380
|
+
}
|
|
3381
|
+
for (const [key, value] of Object.entries(capabilities)) {
|
|
3382
|
+
if (!isRecord(value)) {
|
|
3383
|
+
errors.push(`capabilities.${key} must be an object`);
|
|
3384
|
+
continue;
|
|
3385
|
+
}
|
|
3386
|
+
if (!isSupportedState(value.state)) {
|
|
3387
|
+
errors.push(`capabilities.${key}.state must be one of ${FORTEMI_COMPATIBILITY_STATES.join(", ")}`);
|
|
3388
|
+
}
|
|
3389
|
+
if ("reason_code" in value && typeof value.reason_code !== "string") {
|
|
3390
|
+
errors.push(`capabilities.${key}.reason_code must be a string when present`);
|
|
3391
|
+
}
|
|
3392
|
+
if (value.state !== "available" && !value.reason_code) {
|
|
3393
|
+
warnings.push(`capabilities.${key} is ${String(value.state)} without reason_code`);
|
|
3394
|
+
}
|
|
3395
|
+
}
|
|
3396
|
+
}
|
|
3397
|
+
const links = requireRecord(raw, "links", errors);
|
|
3398
|
+
if (links) {
|
|
3399
|
+
requireString(links, "openapi", "links", errors);
|
|
3400
|
+
requireString(links, "asyncapi", "links", errors);
|
|
3401
|
+
requireString(links, "health", "links", errors);
|
|
3402
|
+
requireString(links, "streaming_health", "links", errors);
|
|
3403
|
+
}
|
|
3404
|
+
return {
|
|
3405
|
+
ok: errors.length === 0,
|
|
3406
|
+
errors,
|
|
3407
|
+
warnings,
|
|
3408
|
+
response: errors.length === 0 ? raw : void 0
|
|
3409
|
+
};
|
|
3410
|
+
}
|
|
3411
|
+
function formatFortemiCompatibilitySummary(response) {
|
|
3412
|
+
const capabilitySummary = Object.entries(response.capabilities).map(([key, value]) => `${key}=${value.state}`).sort().join(", ");
|
|
3413
|
+
return [
|
|
3414
|
+
`Fortemi compatibility ${response.contract_revision}`,
|
|
3415
|
+
`api=${response.api.name}@${response.api.version}`,
|
|
3416
|
+
`deployment=${response.deployment.mode}/${response.deployment.edition}`,
|
|
3417
|
+
`auth=${response.auth.mode}`,
|
|
3418
|
+
`capabilities: ${capabilitySummary}`
|
|
3419
|
+
].join("\n");
|
|
3420
|
+
}
|
|
3421
|
+
async function fetchAndValidateFortemiCompatibility(options = {}) {
|
|
3422
|
+
const fetchImpl = options.fetchImpl ?? globalThis.fetch;
|
|
3423
|
+
if (!fetchImpl) {
|
|
3424
|
+
return {
|
|
3425
|
+
ok: false,
|
|
3426
|
+
url: fortemiCompatibilityUrl(options.baseUrl),
|
|
3427
|
+
errors: ["fetch is not available in this runtime"],
|
|
3428
|
+
warnings: []
|
|
3429
|
+
};
|
|
3430
|
+
}
|
|
3431
|
+
const url = fortemiCompatibilityUrl(options.baseUrl);
|
|
3432
|
+
const controller = new AbortController();
|
|
3433
|
+
const timeout = setTimeout(() => controller.abort(), options.timeoutMs ?? 5e3);
|
|
3434
|
+
try {
|
|
3435
|
+
const response = await fetchImpl(url, {
|
|
3436
|
+
method: "GET",
|
|
3437
|
+
headers: { accept: "application/json" },
|
|
3438
|
+
signal: controller.signal
|
|
3439
|
+
});
|
|
3440
|
+
const status = response.status;
|
|
3441
|
+
const contentType = response.headers.get("content-type") ?? "";
|
|
3442
|
+
if (!response.ok) {
|
|
3443
|
+
return {
|
|
3444
|
+
ok: false,
|
|
3445
|
+
url,
|
|
3446
|
+
status,
|
|
3447
|
+
errors: [`Fortemi compatibility endpoint returned HTTP ${status}`],
|
|
3448
|
+
warnings: []
|
|
3449
|
+
};
|
|
3450
|
+
}
|
|
3451
|
+
if (!contentType.toLowerCase().includes("application/json")) {
|
|
3452
|
+
return {
|
|
3453
|
+
ok: false,
|
|
3454
|
+
url,
|
|
3455
|
+
status,
|
|
3456
|
+
errors: [`Fortemi compatibility endpoint must return JSON; got ${contentType || "unknown content-type"}`],
|
|
3457
|
+
warnings: []
|
|
3458
|
+
};
|
|
3459
|
+
}
|
|
3460
|
+
return {
|
|
3461
|
+
...validateFortemiCompatibilityResponse(await response.json()),
|
|
3462
|
+
url,
|
|
3463
|
+
status
|
|
3464
|
+
};
|
|
3465
|
+
} catch (error) {
|
|
3466
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
3467
|
+
return {
|
|
3468
|
+
ok: false,
|
|
3469
|
+
url,
|
|
3470
|
+
errors: [`Fortemi compatibility endpoint is unreachable: ${message}`],
|
|
3471
|
+
warnings: []
|
|
3472
|
+
};
|
|
3473
|
+
} finally {
|
|
3474
|
+
clearTimeout(timeout);
|
|
3475
|
+
}
|
|
3476
|
+
}
|
|
3477
|
+
|
|
3099
3478
|
// src/service-worker/register.ts
|
|
3100
3479
|
async function registerServiceWorker(swUrl = "/sw.js") {
|
|
3101
3480
|
if (!("serviceWorker" in navigator)) {
|
|
@@ -3862,6 +4241,7 @@ async function getNoteTextWithExtractedAttachments(db, noteId) {
|
|
|
3862
4241
|
' ORDER BY a.position, a.created_at)
|
|
3863
4242
|
FILTER (WHERE a.extracted_text IS NOT NULL AND a.extracted_text <> ''), '') as extracted_text
|
|
3864
4243
|
FROM note_revised_current c
|
|
4244
|
+
JOIN note n ON n.id = c.note_id AND n.deleted_at IS NULL
|
|
3865
4245
|
LEFT JOIN attachment a ON a.note_id = c.note_id AND a.deleted_at IS NULL
|
|
3866
4246
|
WHERE c.note_id = $1
|
|
3867
4247
|
GROUP BY c.content`,
|
|
@@ -4159,7 +4539,7 @@ function conceptTaggingHandler(job, db) {
|
|
|
4159
4539
|
const llmFn2 = getLlmFunction();
|
|
4160
4540
|
if (!llmFn2) return { skipped: true, reason: "no LLM function registered" };
|
|
4161
4541
|
const noteText = await getNoteTextWithExtractedAttachments(db, job.note_id);
|
|
4162
|
-
if (!noteText)
|
|
4542
|
+
if (!noteText) return { skipped: true, reason: "note missing, deleted, or has no content" };
|
|
4163
4543
|
const content = noteText.combined;
|
|
4164
4544
|
const prompt = `Task: Extract 3-5 topic tags from the text below.
|
|
4165
4545
|
Rules:
|
|
@@ -4197,20 +4577,25 @@ Tags:`;
|
|
|
4197
4577
|
function linkingHandler(job, db) {
|
|
4198
4578
|
return (async () => {
|
|
4199
4579
|
const embResult = await db.query(
|
|
4200
|
-
`SELECT id FROM embedding
|
|
4580
|
+
`SELECT e.id FROM embedding e
|
|
4581
|
+
JOIN note n ON n.id = e.note_id AND n.deleted_at IS NULL
|
|
4582
|
+
WHERE e.note_id = $1 LIMIT 1`,
|
|
4201
4583
|
[job.note_id]
|
|
4202
4584
|
);
|
|
4203
4585
|
if (embResult.rows.length === 0) {
|
|
4204
4586
|
return { skipped: true, reason: "no embeddings for this note yet" };
|
|
4205
4587
|
}
|
|
4206
4588
|
const vecResult = await db.query(
|
|
4207
|
-
`SELECT vector::text FROM embedding
|
|
4589
|
+
`SELECT e.vector::text FROM embedding e
|
|
4590
|
+
JOIN note n ON n.id = e.note_id AND n.deleted_at IS NULL
|
|
4591
|
+
WHERE e.note_id = $1 LIMIT 1`,
|
|
4208
4592
|
[job.note_id]
|
|
4209
4593
|
);
|
|
4210
4594
|
if (vecResult.rows.length === 0) return { skipped: true, reason: "no vector found" };
|
|
4211
4595
|
const similar = await db.query(
|
|
4212
4596
|
`SELECT e.note_id, e.vector <=> (SELECT vector FROM embedding WHERE note_id = $1 LIMIT 1) as distance
|
|
4213
4597
|
FROM embedding e
|
|
4598
|
+
JOIN note n ON n.id = e.note_id AND n.deleted_at IS NULL
|
|
4214
4599
|
WHERE e.note_id != $1
|
|
4215
4600
|
ORDER BY distance ASC
|
|
4216
4601
|
LIMIT 5`,
|
|
@@ -5027,14 +5412,16 @@ var AttachmentsRepository = class {
|
|
|
5027
5412
|
/**
|
|
5028
5413
|
* Attach a binary file to a note.
|
|
5029
5414
|
*
|
|
5030
|
-
* If a blob with the same
|
|
5415
|
+
* If a blob with the same BLAKE3 content hash already exists, the existing
|
|
5031
5416
|
* blob row is reused (deduplication). Otherwise a new blob row is inserted
|
|
5032
|
-
* and the raw bytes are written to the BlobStore.
|
|
5417
|
+
* and the raw bytes are written to the BlobStore. The BLAKE3 `content_hash`
|
|
5418
|
+
* (`blake3:<hex>`) matches the server convention and is the key used by the
|
|
5419
|
+
* portable Knowledge-Shard byte sidecar.
|
|
5033
5420
|
*
|
|
5034
5421
|
* Returns the newly created AttachmentRow.
|
|
5035
5422
|
*/
|
|
5036
5423
|
async attach(input) {
|
|
5037
|
-
const contentHash =
|
|
5424
|
+
const contentHash = computeBlobHash(input.data);
|
|
5038
5425
|
const sizeBytes = input.data.length;
|
|
5039
5426
|
let blobId;
|
|
5040
5427
|
const existing = await this.db.query(
|
|
@@ -5148,6 +5535,10 @@ async function manageAttachments(db, blobStore, rawInput) {
|
|
|
5148
5535
|
extractedText: input.extracted_text,
|
|
5149
5536
|
displayName: input.display_name
|
|
5150
5537
|
});
|
|
5538
|
+
if (input.extracted_text) {
|
|
5539
|
+
await enqueueJob(db, { noteId: input.note_id, jobType: "embedding" });
|
|
5540
|
+
await enqueueJob(db, { noteId: input.note_id, jobType: "concept_tagging" });
|
|
5541
|
+
}
|
|
5151
5542
|
return { action: "attach", attachment, size_bytes: data.length };
|
|
5152
5543
|
}
|
|
5153
5544
|
case "list": {
|
|
@@ -5171,7 +5562,12 @@ async function manageAttachments(db, blobStore, rawInput) {
|
|
|
5171
5562
|
}
|
|
5172
5563
|
case "delete": {
|
|
5173
5564
|
if (!input.attachment_id) throw new Error("attachment_id required for delete");
|
|
5565
|
+
const attachment = await repo.get(input.attachment_id);
|
|
5174
5566
|
await repo.delete(input.attachment_id);
|
|
5567
|
+
if (attachment.extracted_text) {
|
|
5568
|
+
await enqueueJob(db, { noteId: attachment.note_id, jobType: "embedding" });
|
|
5569
|
+
await enqueueJob(db, { noteId: attachment.note_id, jobType: "concept_tagging" });
|
|
5570
|
+
}
|
|
5175
5571
|
return { action: "delete", attachment_id: input.attachment_id };
|
|
5176
5572
|
}
|
|
5177
5573
|
}
|
|
@@ -5398,17 +5794,17 @@ async function embeddingGenerationHandler(job, db) {
|
|
|
5398
5794
|
const fn = embedFn;
|
|
5399
5795
|
if (!fn) return { skipped: true, reason: "no embed function registered" };
|
|
5400
5796
|
const noteText = await getNoteTextWithExtractedAttachments(db, job.note_id);
|
|
5401
|
-
if (!noteText)
|
|
5797
|
+
if (!noteText) return { skipped: true, reason: "note missing, deleted, or has no content" };
|
|
5402
5798
|
const content = noteText.combined;
|
|
5403
5799
|
const chunks = chunkText(content);
|
|
5404
5800
|
const embeddings = await fn(chunks);
|
|
5405
|
-
const
|
|
5801
|
+
const vector = averageEmbeddings(embeddings);
|
|
5406
5802
|
const embeddingSets = new EmbeddingSetsRepository(db);
|
|
5407
5803
|
const set = await embeddingSets.ensureDefault();
|
|
5408
5804
|
await embeddingSets.putEmbedding({
|
|
5409
5805
|
note_id: job.note_id,
|
|
5410
5806
|
embedding_set_id: set.id,
|
|
5411
|
-
vector
|
|
5807
|
+
vector
|
|
5412
5808
|
});
|
|
5413
5809
|
return { chunks: chunks.length, embeddings: embeddings.length, setId: set.id };
|
|
5414
5810
|
}
|
|
@@ -5827,13 +6223,13 @@ var OpenAICompatibleProvider = class {
|
|
|
5827
6223
|
throw new Error("Streaming not supported: response body is null");
|
|
5828
6224
|
}
|
|
5829
6225
|
const reader = response.body.getReader();
|
|
5830
|
-
const
|
|
6226
|
+
const decoder6 = new TextDecoder();
|
|
5831
6227
|
let buffer = "";
|
|
5832
6228
|
try {
|
|
5833
6229
|
while (true) {
|
|
5834
6230
|
const { done, value } = await reader.read();
|
|
5835
6231
|
if (done) break;
|
|
5836
|
-
buffer +=
|
|
6232
|
+
buffer += decoder6.decode(value, { stream: true });
|
|
5837
6233
|
const lines = buffer.split("\n");
|
|
5838
6234
|
buffer = lines.pop() ?? "";
|
|
5839
6235
|
for (const line of lines) {
|
|
@@ -6453,6 +6849,23 @@ function createCspReportHandler(onReport) {
|
|
|
6453
6849
|
// src/shard/types.ts
|
|
6454
6850
|
var CURRENT_SHARD_VERSION = "1.0.0";
|
|
6455
6851
|
var SHARD_FORMAT = "matric-shard";
|
|
6852
|
+
function parseVersion(value) {
|
|
6853
|
+
return value.split(".").map((segment) => {
|
|
6854
|
+
const match = segment.match(/^\d+/);
|
|
6855
|
+
return match ? Number.parseInt(match[0], 10) : 0;
|
|
6856
|
+
});
|
|
6857
|
+
}
|
|
6858
|
+
function compareShardVersions(left, right) {
|
|
6859
|
+
const leftParts = parseVersion(left);
|
|
6860
|
+
const rightParts = parseVersion(right);
|
|
6861
|
+
const length = Math.max(leftParts.length, rightParts.length);
|
|
6862
|
+
for (let index = 0; index < length; index += 1) {
|
|
6863
|
+
const leftValue = leftParts[index] ?? 0;
|
|
6864
|
+
const rightValue = rightParts[index] ?? 0;
|
|
6865
|
+
if (leftValue !== rightValue) return leftValue > rightValue ? 1 : -1;
|
|
6866
|
+
}
|
|
6867
|
+
return 0;
|
|
6868
|
+
}
|
|
6456
6869
|
var BLOCK_SIZE = 512;
|
|
6457
6870
|
var USTAR_MAGIC = "ustar\x0000";
|
|
6458
6871
|
function writeString(buf, offset, str, len) {
|
|
@@ -6542,8 +6955,14 @@ function decodeTar(tarData) {
|
|
|
6542
6955
|
}
|
|
6543
6956
|
return files;
|
|
6544
6957
|
}
|
|
6545
|
-
function packTarGz(files) {
|
|
6958
|
+
function packTarGz(files, opts) {
|
|
6546
6959
|
const tarData = encodeTar(files);
|
|
6960
|
+
const cap = opts?.maxDecompressedBytes ?? DEFAULT_MAX_DECOMPRESSED_BYTES;
|
|
6961
|
+
if (tarData.byteLength > cap) {
|
|
6962
|
+
throw new Error(
|
|
6963
|
+
"Refusing to create archive: uncompressed size " + tarData.byteLength + " exceeds cap " + cap + " bytes"
|
|
6964
|
+
);
|
|
6965
|
+
}
|
|
6547
6966
|
return gzipSync(tarData);
|
|
6548
6967
|
}
|
|
6549
6968
|
var DEFAULT_MAX_DECOMPRESSED_BYTES = 256 * 1024 * 1024;
|
|
@@ -6599,7 +7018,8 @@ function noteToShard(note) {
|
|
|
6599
7018
|
title: note.title,
|
|
6600
7019
|
original_content: note.original_content,
|
|
6601
7020
|
revised_content: note.revised_content,
|
|
6602
|
-
|
|
7021
|
+
collection_id: note.collection_id ?? null,
|
|
7022
|
+
...note.attachments?.length ? { attachments: note.attachments } : {},
|
|
6603
7023
|
format: note.format,
|
|
6604
7024
|
source: note.source,
|
|
6605
7025
|
starred: note.is_starred,
|
|
@@ -6611,6 +7031,7 @@ function noteToShard(note) {
|
|
|
6611
7031
|
};
|
|
6612
7032
|
}
|
|
6613
7033
|
function noteFromShard(shard) {
|
|
7034
|
+
const attachments = shard.attachments ?? shard.binary_sources;
|
|
6614
7035
|
return {
|
|
6615
7036
|
id: shard.id,
|
|
6616
7037
|
title: shard.title,
|
|
@@ -6620,7 +7041,8 @@ function noteFromShard(shard) {
|
|
|
6620
7041
|
is_archived: shard.archived,
|
|
6621
7042
|
original_content: shard.original_content,
|
|
6622
7043
|
revised_content: shard.revised_content,
|
|
6623
|
-
|
|
7044
|
+
collection_id: shard.collection_id ?? null,
|
|
7045
|
+
attachments,
|
|
6624
7046
|
tags: shard.tags,
|
|
6625
7047
|
created_at: shard.created_at,
|
|
6626
7048
|
updated_at: shard.updated_at,
|
|
@@ -6632,9 +7054,24 @@ function linkToShard(link) {
|
|
|
6632
7054
|
id: link.id,
|
|
6633
7055
|
from_note_id: link.source_note_id,
|
|
6634
7056
|
to_note_id: link.target_note_id,
|
|
7057
|
+
to_url: null,
|
|
7058
|
+
kind: link.link_type,
|
|
7059
|
+
score: link.confidence,
|
|
7060
|
+
created_at: toISOString(link.created_at),
|
|
7061
|
+
metadata: null
|
|
7062
|
+
};
|
|
7063
|
+
}
|
|
7064
|
+
function urlLinkToShard(link) {
|
|
7065
|
+
const metadata = typeof link.metadata_json === "string" ? JSON.parse(link.metadata_json) : link.metadata_json ?? null;
|
|
7066
|
+
return {
|
|
7067
|
+
id: link.id,
|
|
7068
|
+
from_note_id: link.source_note_id,
|
|
7069
|
+
to_note_id: null,
|
|
7070
|
+
to_url: link.to_url,
|
|
6635
7071
|
kind: link.link_type,
|
|
6636
7072
|
score: link.confidence,
|
|
6637
|
-
created_at: toISOString(link.created_at)
|
|
7073
|
+
created_at: toISOString(link.created_at),
|
|
7074
|
+
metadata
|
|
6638
7075
|
};
|
|
6639
7076
|
}
|
|
6640
7077
|
function linkFromShard(shard) {
|
|
@@ -6642,9 +7079,11 @@ function linkFromShard(shard) {
|
|
|
6642
7079
|
id: shard.id,
|
|
6643
7080
|
source_note_id: shard.from_note_id,
|
|
6644
7081
|
target_note_id: shard.to_note_id,
|
|
7082
|
+
to_url: shard.to_url,
|
|
6645
7083
|
link_type: shard.kind,
|
|
6646
7084
|
confidence: shard.score,
|
|
6647
|
-
created_at: shard.created_at
|
|
7085
|
+
created_at: shard.created_at,
|
|
7086
|
+
metadata: shard.metadata
|
|
6648
7087
|
};
|
|
6649
7088
|
}
|
|
6650
7089
|
function collectionToShard(collection, noteCount) {
|
|
@@ -6672,14 +7111,31 @@ function tagsToShard(allTags) {
|
|
|
6672
7111
|
created_at: toISOString(t.created_at)
|
|
6673
7112
|
}));
|
|
6674
7113
|
}
|
|
6675
|
-
function
|
|
6676
|
-
return
|
|
7114
|
+
function templateToShard(template) {
|
|
7115
|
+
return {
|
|
7116
|
+
id: template.id,
|
|
7117
|
+
name: template.name,
|
|
7118
|
+
description: template.description,
|
|
7119
|
+
content: template.content,
|
|
7120
|
+
format: template.format,
|
|
7121
|
+
default_tags: Array.isArray(template.default_tags) ? template.default_tags : JSON.parse(template.default_tags),
|
|
7122
|
+
collection_id: template.collection_id,
|
|
7123
|
+
created_at: toISOString(template.created_at),
|
|
7124
|
+
updated_at: toISOString(template.updated_at)
|
|
7125
|
+
};
|
|
6677
7126
|
}
|
|
6678
7127
|
function embeddingSetToShard(set) {
|
|
7128
|
+
const name = set.name ?? set.model_name;
|
|
6679
7129
|
return {
|
|
6680
7130
|
id: set.id,
|
|
6681
|
-
name
|
|
7131
|
+
name,
|
|
7132
|
+
slug: set.slug ?? slugifyEmbeddingSet(name),
|
|
7133
|
+
description: set.description ?? null,
|
|
6682
7134
|
purpose: set.purpose ?? null,
|
|
7135
|
+
document_count: set.document_count ?? 0,
|
|
7136
|
+
embedding_count: set.embedding_count ?? 0,
|
|
7137
|
+
is_system: set.is_system ?? false,
|
|
7138
|
+
keywords: jsonStringArray(set.keywords_json),
|
|
6683
7139
|
model: set.model_name,
|
|
6684
7140
|
dimension: set.dimensions,
|
|
6685
7141
|
kind: set.kind ?? "physical",
|
|
@@ -6694,11 +7150,17 @@ function embeddingSetToShard(set) {
|
|
|
6694
7150
|
updated_at: set.updated_at ? toISOString(set.updated_at) : void 0
|
|
6695
7151
|
};
|
|
6696
7152
|
}
|
|
6697
|
-
function embeddingSetFromShard(shard) {
|
|
7153
|
+
function embeddingSetFromShard(shard, fallbackCreatedAt) {
|
|
6698
7154
|
return {
|
|
6699
7155
|
id: shard.id,
|
|
6700
7156
|
name: shard.name ?? shard.model,
|
|
7157
|
+
slug: shard.slug ?? null,
|
|
7158
|
+
description: shard.description ?? null,
|
|
6701
7159
|
purpose: shard.purpose ?? null,
|
|
7160
|
+
document_count: shard.document_count ?? null,
|
|
7161
|
+
embedding_count: shard.embedding_count ?? null,
|
|
7162
|
+
is_system: shard.is_system ?? false,
|
|
7163
|
+
keywords_json: jsonString(shard.keywords ?? []),
|
|
6702
7164
|
model_name: shard.model,
|
|
6703
7165
|
dimensions: shard.dimension,
|
|
6704
7166
|
kind: shard.kind ?? "physical",
|
|
@@ -6709,7 +7171,7 @@ function embeddingSetFromShard(shard) {
|
|
|
6709
7171
|
compatibility_json: jsonString(shard.compatibility),
|
|
6710
7172
|
materialization_json: jsonString(shard.materialization),
|
|
6711
7173
|
freshness_json: jsonString(shard.freshness),
|
|
6712
|
-
created_at: shard.created_at,
|
|
7174
|
+
created_at: shard.created_at ?? fallbackCreatedAt,
|
|
6713
7175
|
updated_at: shard.updated_at ?? null
|
|
6714
7176
|
};
|
|
6715
7177
|
}
|
|
@@ -6717,15 +7179,32 @@ function embeddingSetMemberToShard(member) {
|
|
|
6717
7179
|
return {
|
|
6718
7180
|
embedding_set_id: member.embedding_set_id,
|
|
6719
7181
|
note_id: member.note_id,
|
|
6720
|
-
|
|
7182
|
+
membership_type: member.membership_type ?? "materialized",
|
|
7183
|
+
added_at: toISOString(member.added_at ?? /* @__PURE__ */ new Date()),
|
|
7184
|
+
added_by: member.added_by ?? null
|
|
7185
|
+
};
|
|
7186
|
+
}
|
|
7187
|
+
function embeddingConfigToShard(config) {
|
|
7188
|
+
return {
|
|
7189
|
+
id: config.id,
|
|
7190
|
+
name: config.name,
|
|
7191
|
+
description: config.description ?? null,
|
|
7192
|
+
model: config.model,
|
|
7193
|
+
dimension: config.dimension,
|
|
7194
|
+
chunk_size: config.chunk_size,
|
|
7195
|
+
chunk_overlap: config.chunk_overlap,
|
|
7196
|
+
is_default: config.is_default
|
|
6721
7197
|
};
|
|
6722
7198
|
}
|
|
6723
7199
|
function embeddingToShard(emb) {
|
|
6724
7200
|
return {
|
|
6725
7201
|
id: emb.id,
|
|
6726
7202
|
note_id: emb.note_id,
|
|
6727
|
-
|
|
7203
|
+
chunk_index: emb.chunk_index ?? 0,
|
|
7204
|
+
text: emb.text ?? "",
|
|
6728
7205
|
vector: typeof emb.vector === "string" ? parseVector2(emb.vector) : emb.vector,
|
|
7206
|
+
model: emb.model ?? emb.model_name ?? "unknown",
|
|
7207
|
+
embedding_set_id: emb.embedding_set_id,
|
|
6729
7208
|
created_at: toISOString(emb.created_at)
|
|
6730
7209
|
};
|
|
6731
7210
|
}
|
|
@@ -6733,9 +7212,12 @@ function embeddingFromShard(shard) {
|
|
|
6733
7212
|
return {
|
|
6734
7213
|
id: shard.id,
|
|
6735
7214
|
note_id: shard.note_id,
|
|
6736
|
-
embedding_set_id: shard.embedding_set_id,
|
|
7215
|
+
embedding_set_id: shard.embedding_set_id ?? null,
|
|
7216
|
+
chunk_index: shard.chunk_index,
|
|
7217
|
+
text: shard.text,
|
|
6737
7218
|
vector: `[${shard.vector.join(",")}]`,
|
|
6738
|
-
|
|
7219
|
+
model: shard.model,
|
|
7220
|
+
created_at: shard.created_at ?? null
|
|
6739
7221
|
};
|
|
6740
7222
|
}
|
|
6741
7223
|
function skosSchemeToShard(scheme) {
|
|
@@ -6807,6 +7289,15 @@ function parseJsonObjectField(value) {
|
|
|
6807
7289
|
const parsed = JSON.parse(value);
|
|
6808
7290
|
return parsed && typeof parsed === "object" && !Array.isArray(parsed) ? parsed : null;
|
|
6809
7291
|
}
|
|
7292
|
+
function jsonStringArray(value) {
|
|
7293
|
+
if (value == null) return [];
|
|
7294
|
+
if (Array.isArray(value)) return value.map(String);
|
|
7295
|
+
if (typeof value === "string") {
|
|
7296
|
+
const parsed = JSON.parse(value);
|
|
7297
|
+
return Array.isArray(parsed) ? parsed.map(String) : [];
|
|
7298
|
+
}
|
|
7299
|
+
return [];
|
|
7300
|
+
}
|
|
6810
7301
|
function jsonObject2(value) {
|
|
6811
7302
|
if (value == null) return null;
|
|
6812
7303
|
if (typeof value === "string") return JSON.parse(value);
|
|
@@ -6815,6 +7306,34 @@ function jsonObject2(value) {
|
|
|
6815
7306
|
function jsonString(value) {
|
|
6816
7307
|
return value == null ? null : JSON.stringify(value);
|
|
6817
7308
|
}
|
|
7309
|
+
function slugifyEmbeddingSet(value) {
|
|
7310
|
+
const slug = value.trim().toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "");
|
|
7311
|
+
return slug || "embedding-set";
|
|
7312
|
+
}
|
|
7313
|
+
|
|
7314
|
+
// src/shard/blob-sidecar.ts
|
|
7315
|
+
var SIDECAR_PREFIX = "blobs/";
|
|
7316
|
+
function blobChecksumToHex(checksum) {
|
|
7317
|
+
const sep = checksum.indexOf(":");
|
|
7318
|
+
return sep >= 0 ? checksum.slice(sep + 1) : checksum;
|
|
7319
|
+
}
|
|
7320
|
+
function sidecarEntryName(checksum) {
|
|
7321
|
+
return SIDECAR_PREFIX + blobChecksumToHex(checksum);
|
|
7322
|
+
}
|
|
7323
|
+
function isSidecarEntry(name) {
|
|
7324
|
+
if (!name.startsWith(SIDECAR_PREFIX)) return false;
|
|
7325
|
+
const rest = name.slice(SIDECAR_PREFIX.length);
|
|
7326
|
+
return rest.length > 0 && !rest.includes("/");
|
|
7327
|
+
}
|
|
7328
|
+
function collectSidecarBlobs(files) {
|
|
7329
|
+
const blobs = /* @__PURE__ */ new Map();
|
|
7330
|
+
for (const [name, bytes] of files) {
|
|
7331
|
+
if (isSidecarEntry(name)) {
|
|
7332
|
+
blobs.set(name.slice(SIDECAR_PREFIX.length), bytes);
|
|
7333
|
+
}
|
|
7334
|
+
}
|
|
7335
|
+
return blobs;
|
|
7336
|
+
}
|
|
6818
7337
|
|
|
6819
7338
|
// src/shard/shard-export.ts
|
|
6820
7339
|
var encoder = new TextEncoder();
|
|
@@ -6836,7 +7355,8 @@ async function exportShard(db, options) {
|
|
|
6836
7355
|
noteQuery = `SELECT n.id, n.title, n.format, n.source, n.is_starred, n.is_archived,
|
|
6837
7356
|
n.created_at, n.updated_at, n.deleted_at,
|
|
6838
7357
|
o.content as original_content,
|
|
6839
|
-
c.content as revised_content
|
|
7358
|
+
c.content as revised_content,
|
|
7359
|
+
$1::text as collection_id
|
|
6840
7360
|
FROM note n
|
|
6841
7361
|
LEFT JOIN note_original o ON o.note_id = n.id
|
|
6842
7362
|
LEFT JOIN note_revised_current c ON c.note_id = n.id
|
|
@@ -6848,7 +7368,14 @@ async function exportShard(db, options) {
|
|
|
6848
7368
|
noteQuery = `SELECT n.id, n.title, n.format, n.source, n.is_starred, n.is_archived,
|
|
6849
7369
|
n.created_at, n.updated_at, n.deleted_at,
|
|
6850
7370
|
o.content as original_content,
|
|
6851
|
-
c.content as revised_content
|
|
7371
|
+
c.content as revised_content,
|
|
7372
|
+
(
|
|
7373
|
+
SELECT cn.collection_id
|
|
7374
|
+
FROM collection_note cn
|
|
7375
|
+
WHERE cn.note_id = n.id
|
|
7376
|
+
ORDER BY cn.position, cn.added_at
|
|
7377
|
+
LIMIT 1
|
|
7378
|
+
) as collection_id
|
|
6852
7379
|
FROM note n
|
|
6853
7380
|
LEFT JOIN note_original o ON o.note_id = n.id
|
|
6854
7381
|
LEFT JOIN note_revised_current c ON c.note_id = n.id
|
|
@@ -6860,7 +7387,14 @@ async function exportShard(db, options) {
|
|
|
6860
7387
|
noteQuery = `SELECT n.id, n.title, n.format, n.source, n.is_starred, n.is_archived,
|
|
6861
7388
|
n.created_at, n.updated_at, n.deleted_at,
|
|
6862
7389
|
o.content as original_content,
|
|
6863
|
-
c.content as revised_content
|
|
7390
|
+
c.content as revised_content,
|
|
7391
|
+
(
|
|
7392
|
+
SELECT cn.collection_id
|
|
7393
|
+
FROM collection_note cn
|
|
7394
|
+
WHERE cn.note_id = n.id
|
|
7395
|
+
ORDER BY cn.position, cn.added_at
|
|
7396
|
+
LIMIT 1
|
|
7397
|
+
) as collection_id
|
|
6864
7398
|
FROM note n
|
|
6865
7399
|
LEFT JOIN note_original o ON o.note_id = n.id
|
|
6866
7400
|
LEFT JOIN note_revised_current c ON c.note_id = n.id
|
|
@@ -6892,26 +7426,28 @@ async function exportShard(db, options) {
|
|
|
6892
7426
|
WHERE a.deleted_at IS NULL
|
|
6893
7427
|
ORDER BY a.note_id, a.position, a.created_at`
|
|
6894
7428
|
);
|
|
6895
|
-
const
|
|
7429
|
+
const attachmentsByNote = /* @__PURE__ */ new Map();
|
|
6896
7430
|
for (const row of attachmentRows.rows) {
|
|
6897
7431
|
const source = {
|
|
6898
|
-
extracted_text: row.extracted_text
|
|
7432
|
+
extracted_text: row.extracted_text,
|
|
6899
7433
|
attachment: {
|
|
7434
|
+
// `path` is the display filename per the binary-attachment projection
|
|
7435
|
+
// contract — never the physical storage key (`storage_path`).
|
|
6900
7436
|
id: row.id,
|
|
6901
|
-
path: row.
|
|
7437
|
+
path: row.filename,
|
|
6902
7438
|
mime: row.mime_type,
|
|
6903
7439
|
checksum: row.content_hash,
|
|
6904
7440
|
bytes: Number(row.size_bytes)
|
|
6905
7441
|
}
|
|
6906
7442
|
};
|
|
6907
|
-
const sources =
|
|
7443
|
+
const sources = attachmentsByNote.get(row.note_id) ?? [];
|
|
6908
7444
|
sources.push(source);
|
|
6909
|
-
|
|
7445
|
+
attachmentsByNote.set(row.note_id, sources);
|
|
6910
7446
|
}
|
|
6911
7447
|
const notes = noteRows.rows.map((row) => ({
|
|
6912
7448
|
...row,
|
|
6913
7449
|
tags: tagsByNote.get(row.id) ?? [],
|
|
6914
|
-
|
|
7450
|
+
attachments: attachmentsByNote.get(row.id)
|
|
6915
7451
|
}));
|
|
6916
7452
|
const exportedNoteIds = new Set(notes.map((n) => n.id));
|
|
6917
7453
|
const shardNotes = notes.map((n) => noteToShard(n));
|
|
@@ -6922,7 +7458,7 @@ async function exportShard(db, options) {
|
|
|
6922
7458
|
for (let offset = 0; offset < shardNotes.length; offset += clusterSize) {
|
|
6923
7459
|
const slice = shardNotes.slice(offset, offset + clusterSize);
|
|
6924
7460
|
const href = `notes/${String(offset).padStart(6, "0")}.jsonl`;
|
|
6925
|
-
clusters.push({ href, offset
|
|
7461
|
+
clusters.push({ href, offset });
|
|
6926
7462
|
files.set(href, encoder.encode(slice.map((n) => JSON.stringify(n)).join("\n")));
|
|
6927
7463
|
}
|
|
6928
7464
|
layout = { clusters: { notes: clusters } };
|
|
@@ -6963,14 +7499,27 @@ async function exportShard(db, options) {
|
|
|
6963
7499
|
files.set("tags.json", encoder.encode(JSON.stringify(shardTags)));
|
|
6964
7500
|
components.push("tags");
|
|
6965
7501
|
counts.tags = shardTags.length;
|
|
7502
|
+
const templateRows = await db.query(`SELECT * FROM template ORDER BY created_at, id`);
|
|
7503
|
+
if (templateRows.rows.length > 0) {
|
|
7504
|
+
const shardTemplates = templateRows.rows.map((template) => templateToShard(template));
|
|
7505
|
+
files.set("templates.json", encoder.encode(JSON.stringify(shardTemplates)));
|
|
7506
|
+
components.push("templates");
|
|
7507
|
+
counts.templates = shardTemplates.length;
|
|
7508
|
+
}
|
|
6966
7509
|
const linkRows = await db.query(
|
|
6967
7510
|
`SELECT * FROM link WHERE deleted_at IS NULL ORDER BY created_at`
|
|
6968
7511
|
);
|
|
6969
7512
|
const filteredLinks = options?.tag || options?.collectionId ? linkRows.rows.filter((l) => exportedNoteIds.has(l.source_note_id) && exportedNoteIds.has(l.target_note_id)) : linkRows.rows;
|
|
6970
|
-
const
|
|
7513
|
+
const urlLinkRows = await db.query(`SELECT * FROM link_url_target WHERE deleted_at IS NULL ORDER BY created_at`);
|
|
7514
|
+
const filteredUrlLinks = options?.tag || options?.collectionId ? urlLinkRows.rows.filter((l) => exportedNoteIds.has(l.source_note_id)) : urlLinkRows.rows;
|
|
7515
|
+
const shardLinks = [
|
|
7516
|
+
...filteredLinks.map((l) => linkToShard(l)),
|
|
7517
|
+
...filteredUrlLinks.map((l) => urlLinkToShard(l))
|
|
7518
|
+
];
|
|
7519
|
+
const linksJsonl = shardLinks.map((l) => JSON.stringify(l)).join("\n");
|
|
6971
7520
|
files.set("links.jsonl", encoder.encode(linksJsonl));
|
|
6972
7521
|
components.push("links");
|
|
6973
|
-
counts.links =
|
|
7522
|
+
counts.links = shardLinks.length;
|
|
6974
7523
|
const allNoteSkosRows = await db.query(`SELECT * FROM note_skos_tag ORDER BY created_at`);
|
|
6975
7524
|
const filteredNoteSkosRows = isFiltered ? allNoteSkosRows.rows.filter((row) => exportedNoteIds.has(row.note_id)) : allNoteSkosRows.rows;
|
|
6976
7525
|
const referencedConceptIds = new Set(filteredNoteSkosRows.map((row) => row.concept_id));
|
|
@@ -7011,9 +7560,27 @@ async function exportShard(db, options) {
|
|
|
7011
7560
|
const setScoped = embeddingSetIds.length > 0;
|
|
7012
7561
|
const includeMaterializedSelectors = options.includeMaterializedSelectors === true;
|
|
7013
7562
|
const embSetRows = await db.query(
|
|
7014
|
-
`SELECT
|
|
7015
|
-
|
|
7016
|
-
|
|
7563
|
+
`SELECT
|
|
7564
|
+
es.id, es.name, es.slug, es.description, es.purpose,
|
|
7565
|
+
COALESCE(es.document_count, member_counts.document_count, 0)::int AS document_count,
|
|
7566
|
+
COALESCE(es.embedding_count, embedding_counts.embedding_count, 0)::int AS embedding_count,
|
|
7567
|
+
es.is_system, es.keywords_json,
|
|
7568
|
+
es.model_name, es.dimensions, es.kind, es.mode, es.truncate_dimension,
|
|
7569
|
+
es.criteria_json, es.source_json, es.compatibility_json, es.materialization_json,
|
|
7570
|
+
es.freshness_json, es.created_at, es.updated_at
|
|
7571
|
+
FROM embedding_set es
|
|
7572
|
+
LEFT JOIN (
|
|
7573
|
+
SELECT embedding_set_id, COUNT(*)::int AS document_count
|
|
7574
|
+
FROM embedding_set_member
|
|
7575
|
+
GROUP BY embedding_set_id
|
|
7576
|
+
) member_counts ON member_counts.embedding_set_id = es.id
|
|
7577
|
+
LEFT JOIN (
|
|
7578
|
+
SELECT embedding_set_id, COUNT(*)::int AS embedding_count
|
|
7579
|
+
FROM embedding
|
|
7580
|
+
GROUP BY embedding_set_id
|
|
7581
|
+
) embedding_counts ON embedding_counts.embedding_set_id = es.id
|
|
7582
|
+
${setScoped ? "WHERE es.id = ANY($1)" : ""}
|
|
7583
|
+
ORDER BY es.created_at`,
|
|
7017
7584
|
setScoped ? [embeddingSetIds] : []
|
|
7018
7585
|
);
|
|
7019
7586
|
const exportedSetIds = new Set(embSetRows.rows.map((row) => row.id));
|
|
@@ -7028,6 +7595,17 @@ async function exportShard(db, options) {
|
|
|
7028
7595
|
files.set("embedding_sets.json", encoder.encode(JSON.stringify(shardEmbSets)));
|
|
7029
7596
|
components.push("embedding_sets");
|
|
7030
7597
|
counts.embedding_sets = shardEmbSets.length;
|
|
7598
|
+
const embeddingConfigRows = await db.query(
|
|
7599
|
+
`SELECT id, name, description, model, dimension, chunk_size, chunk_overlap, is_default
|
|
7600
|
+
FROM embedding_config
|
|
7601
|
+
ORDER BY name, id`
|
|
7602
|
+
);
|
|
7603
|
+
if (embeddingConfigRows.rows.length > 0) {
|
|
7604
|
+
const shardEmbeddingConfigs = embeddingConfigRows.rows.map((row) => embeddingConfigToShard(row));
|
|
7605
|
+
files.set("embedding_configs.json", encoder.encode(JSON.stringify(shardEmbeddingConfigs)));
|
|
7606
|
+
components.push("embedding_configs");
|
|
7607
|
+
counts.embedding_configs = shardEmbeddingConfigs.length;
|
|
7608
|
+
}
|
|
7031
7609
|
const embMemberRows = await db.query(
|
|
7032
7610
|
`SELECT * FROM embedding_set_member
|
|
7033
7611
|
${setScoped ? "WHERE embedding_set_id = ANY($1)" : ""}`,
|
|
@@ -7041,12 +7619,24 @@ async function exportShard(db, options) {
|
|
|
7041
7619
|
components.push("embedding_set_members");
|
|
7042
7620
|
counts.embedding_set_members = scopedEmbMemberRows.length;
|
|
7043
7621
|
const embRows = await db.query(
|
|
7044
|
-
`SELECT
|
|
7045
|
-
|
|
7046
|
-
|
|
7622
|
+
`SELECT
|
|
7623
|
+
e.id, e.note_id, e.embedding_set_id, e.chunk_index,
|
|
7624
|
+
COALESCE(NULLIF(e.text, ''), nrc.content, no.content, '') AS text,
|
|
7625
|
+
e.vector,
|
|
7626
|
+
COALESCE(e.model, es.model_name) AS model,
|
|
7627
|
+
es.model_name,
|
|
7628
|
+
e.created_at
|
|
7629
|
+
FROM embedding e
|
|
7630
|
+
JOIN embedding_set es ON es.id = e.embedding_set_id
|
|
7631
|
+
LEFT JOIN note_revised_current nrc ON nrc.note_id = e.note_id
|
|
7632
|
+
LEFT JOIN note_original no ON no.note_id = e.note_id
|
|
7633
|
+
${setScoped ? "WHERE e.embedding_set_id = ANY($1)" : ""}
|
|
7634
|
+
ORDER BY e.created_at`,
|
|
7047
7635
|
setScoped ? [embeddingSetIds] : []
|
|
7048
7636
|
);
|
|
7049
|
-
const memberEmbeddingIds = new Set(
|
|
7637
|
+
const memberEmbeddingIds = new Set(
|
|
7638
|
+
scopedEmbMemberRows.map((member) => member.embedding_id).filter((id) => Boolean(id))
|
|
7639
|
+
);
|
|
7050
7640
|
const scopedEmbRows = embRows.rows.filter(
|
|
7051
7641
|
(embedding) => exportedSetIds.has(embedding.embedding_set_id) && exportedNoteIds.has(embedding.note_id) && (memberEmbeddingIds.size === 0 || memberEmbeddingIds.has(embedding.id))
|
|
7052
7642
|
);
|
|
@@ -7162,14 +7752,37 @@ async function exportShard(db, options) {
|
|
|
7162
7752
|
counts,
|
|
7163
7753
|
checksums,
|
|
7164
7754
|
min_reader_version: "1.0.0",
|
|
7755
|
+
migrated_from: null,
|
|
7756
|
+
migration_history: [],
|
|
7165
7757
|
...layout ? { layout } : {}
|
|
7166
7758
|
};
|
|
7167
7759
|
files.set("manifest.json", encoder.encode(JSON.stringify(manifest, null, 2)));
|
|
7760
|
+
if (options?.includeBlobs && options.blobStore) {
|
|
7761
|
+
const packed = /* @__PURE__ */ new Set();
|
|
7762
|
+
for (const row of attachmentRows.rows) {
|
|
7763
|
+
const checksum = row.content_hash;
|
|
7764
|
+
if (packed.has(checksum)) continue;
|
|
7765
|
+
packed.add(checksum);
|
|
7766
|
+
const bytes = await options.blobStore.read(checksum);
|
|
7767
|
+
if (bytes) files.set(sidecarEntryName(checksum), bytes);
|
|
7768
|
+
}
|
|
7769
|
+
}
|
|
7168
7770
|
return packTarGz(files);
|
|
7169
7771
|
}
|
|
7170
7772
|
|
|
7171
|
-
// src/shard/
|
|
7773
|
+
// src/shard/parse.ts
|
|
7172
7774
|
var decoder = new TextDecoder();
|
|
7775
|
+
function parseJsonlBytes(data) {
|
|
7776
|
+
if (!data || data.byteLength === 0) return [];
|
|
7777
|
+
return decoder.decode(data).split("\n").filter((line) => line.trim()).map((line) => JSON.parse(line));
|
|
7778
|
+
}
|
|
7779
|
+
function parseJsonArrayBytes(data) {
|
|
7780
|
+
if (!data || data.byteLength === 0) return [];
|
|
7781
|
+
return JSON.parse(decoder.decode(data));
|
|
7782
|
+
}
|
|
7783
|
+
|
|
7784
|
+
// src/shard/shard-import.ts
|
|
7785
|
+
var decoder2 = new TextDecoder();
|
|
7173
7786
|
var DEFAULT_BATCH_SIZE = 250;
|
|
7174
7787
|
async function yieldToEventLoop2() {
|
|
7175
7788
|
const scheduler = globalThis.scheduler;
|
|
@@ -7191,14 +7804,17 @@ async function importShard(db, data, options) {
|
|
|
7191
7804
|
const report = options?.onProgress;
|
|
7192
7805
|
const warnings = [];
|
|
7193
7806
|
const errors = [];
|
|
7194
|
-
let
|
|
7195
|
-
let
|
|
7807
|
+
let importedAttachmentReferenceCount = 0;
|
|
7808
|
+
let notesWithImportedAttachmentReferences = 0;
|
|
7809
|
+
const blobsToHydrate = /* @__PURE__ */ new Map();
|
|
7196
7810
|
const counts = {
|
|
7197
7811
|
notes: 0,
|
|
7198
7812
|
collections: 0,
|
|
7813
|
+
templates: 0,
|
|
7199
7814
|
tags: 0,
|
|
7200
7815
|
links: 0,
|
|
7201
7816
|
embedding_sets: 0,
|
|
7817
|
+
embedding_configs: 0,
|
|
7202
7818
|
embedding_set_members: 0,
|
|
7203
7819
|
embeddings: 0,
|
|
7204
7820
|
skos_schemes: 0,
|
|
@@ -7242,7 +7858,10 @@ async function importShard(db, data, options) {
|
|
|
7242
7858
|
}
|
|
7243
7859
|
let manifest;
|
|
7244
7860
|
try {
|
|
7245
|
-
manifest = JSON.parse(
|
|
7861
|
+
manifest = JSON.parse(decoder2.decode(manifestData));
|
|
7862
|
+
if (!manifest || typeof manifest !== "object" || Array.isArray(manifest)) {
|
|
7863
|
+
throw new Error("manifest must be a JSON object");
|
|
7864
|
+
}
|
|
7246
7865
|
} catch {
|
|
7247
7866
|
return {
|
|
7248
7867
|
success: false,
|
|
@@ -7253,7 +7872,7 @@ async function importShard(db, data, options) {
|
|
|
7253
7872
|
duration_ms: performance.now() - start
|
|
7254
7873
|
};
|
|
7255
7874
|
}
|
|
7256
|
-
if (manifest.min_reader_version && manifest.min_reader_version >
|
|
7875
|
+
if (manifest.min_reader_version && compareShardVersions(manifest.min_reader_version, CURRENT_SHARD_VERSION) > 0) {
|
|
7257
7876
|
return {
|
|
7258
7877
|
success: false,
|
|
7259
7878
|
counts,
|
|
@@ -7266,6 +7885,16 @@ async function importShard(db, data, options) {
|
|
|
7266
7885
|
};
|
|
7267
7886
|
}
|
|
7268
7887
|
report?.({ phase: "validate", done: 0, total: 1 });
|
|
7888
|
+
if (!manifest.checksums || typeof manifest.checksums !== "object") {
|
|
7889
|
+
return {
|
|
7890
|
+
success: false,
|
|
7891
|
+
counts,
|
|
7892
|
+
skipped,
|
|
7893
|
+
warnings,
|
|
7894
|
+
errors: ["Invalid manifest.json: checksums must be an object"],
|
|
7895
|
+
duration_ms: performance.now() - start
|
|
7896
|
+
};
|
|
7897
|
+
}
|
|
7269
7898
|
const checksumResult = await validateChecksums(manifest.checksums, files);
|
|
7270
7899
|
if (!checksumResult.valid) {
|
|
7271
7900
|
return {
|
|
@@ -7278,25 +7907,61 @@ async function importShard(db, data, options) {
|
|
|
7278
7907
|
};
|
|
7279
7908
|
}
|
|
7280
7909
|
report?.({ phase: "validate", done: 1, total: 1 });
|
|
7910
|
+
const sidecarBlobs = options?.blobStore ? collectSidecarBlobs(files) : null;
|
|
7281
7911
|
const noteClusters = manifest.layout?.clusters?.notes;
|
|
7282
|
-
const
|
|
7283
|
-
|
|
7284
|
-
|
|
7285
|
-
|
|
7286
|
-
|
|
7287
|
-
|
|
7288
|
-
|
|
7289
|
-
|
|
7290
|
-
|
|
7291
|
-
|
|
7292
|
-
|
|
7293
|
-
|
|
7294
|
-
|
|
7295
|
-
|
|
7296
|
-
|
|
7297
|
-
|
|
7298
|
-
|
|
7299
|
-
|
|
7912
|
+
const parseComponent = (name, parse) => {
|
|
7913
|
+
try {
|
|
7914
|
+
return parse();
|
|
7915
|
+
} catch (err) {
|
|
7916
|
+
throw new Error(`${name}: ${err instanceof Error ? err.message : String(err)}`);
|
|
7917
|
+
}
|
|
7918
|
+
};
|
|
7919
|
+
let parsedNotes;
|
|
7920
|
+
let parsedCollections;
|
|
7921
|
+
let parsedTemplates;
|
|
7922
|
+
let parsedLinks;
|
|
7923
|
+
let parsedEmbSets;
|
|
7924
|
+
let parsedEmbConfigs;
|
|
7925
|
+
let parsedEmbMembers;
|
|
7926
|
+
let parsedEmbeddings;
|
|
7927
|
+
let parsedSkosSchemes;
|
|
7928
|
+
let parsedSkosConcepts;
|
|
7929
|
+
let parsedSkosRelations;
|
|
7930
|
+
let parsedNoteSkosTags;
|
|
7931
|
+
let parsedProvenanceEdges;
|
|
7932
|
+
let parsedGraphSources;
|
|
7933
|
+
let parsedGraphEdges;
|
|
7934
|
+
let parsedCommunitySets;
|
|
7935
|
+
let parsedCommunityAssignments;
|
|
7936
|
+
try {
|
|
7937
|
+
parsedNotes = parseComponent("notes", () => noteClusters && noteClusters.length > 0 ? [...noteClusters].sort((a, b) => a.offset - b.offset).flatMap((ref) => parseJsonlBytes(files.get(ref.href))) : parseJsonlBytes(files.get("notes.jsonl")));
|
|
7938
|
+
parsedCollections = parseComponent("collections.json", () => parseJsonArrayBytes(files.get("collections.json")));
|
|
7939
|
+
parseComponent("tags.json", () => parseJsonArrayBytes(files.get("tags.json")));
|
|
7940
|
+
parsedTemplates = parseComponent("templates.json", () => parseJsonArrayBytes(files.get("templates.json")));
|
|
7941
|
+
parsedLinks = parseComponent("links.jsonl", () => parseJsonlBytes(files.get("links.jsonl")));
|
|
7942
|
+
parsedEmbSets = parseComponent("embedding_sets.json", () => parseJsonArrayBytes(files.get("embedding_sets.json")));
|
|
7943
|
+
parsedEmbConfigs = parseComponent("embedding_configs.json", () => parseJsonArrayBytes(files.get("embedding_configs.json")));
|
|
7944
|
+
parsedEmbMembers = parseComponent("embedding_set_members.jsonl", () => parseJsonlBytes(files.get("embedding_set_members.jsonl")));
|
|
7945
|
+
parsedEmbeddings = parseComponent("embeddings.jsonl", () => parseJsonlBytes(files.get("embeddings.jsonl")));
|
|
7946
|
+
parsedSkosSchemes = parseComponent("skos_schemes.json", () => parseJsonArrayBytes(files.get("skos_schemes.json")));
|
|
7947
|
+
parsedSkosConcepts = parseComponent("skos_concepts.json", () => parseJsonArrayBytes(files.get("skos_concepts.json")));
|
|
7948
|
+
parsedSkosRelations = parseComponent("skos_relations.jsonl", () => parseJsonlBytes(files.get("skos_relations.jsonl")));
|
|
7949
|
+
parsedNoteSkosTags = parseComponent("note_skos_tags.jsonl", () => parseJsonlBytes(files.get("note_skos_tags.jsonl")));
|
|
7950
|
+
parsedProvenanceEdges = parseComponent("provenance_edges.jsonl", () => parseJsonlBytes(files.get("provenance_edges.jsonl")));
|
|
7951
|
+
parsedGraphSources = parseComponent("graph_sources.json", () => parseJsonArrayBytes(files.get("graph_sources.json")));
|
|
7952
|
+
parsedGraphEdges = parseComponent("graph_edges.jsonl", () => parseJsonlBytes(files.get("graph_edges.jsonl")));
|
|
7953
|
+
parsedCommunitySets = parseComponent("communities.json", () => parseJsonArrayBytes(files.get("communities.json")));
|
|
7954
|
+
parsedCommunityAssignments = parseComponent("community_assignments.jsonl", () => parseJsonlBytes(files.get("community_assignments.jsonl")));
|
|
7955
|
+
} catch (err) {
|
|
7956
|
+
return {
|
|
7957
|
+
success: false,
|
|
7958
|
+
counts,
|
|
7959
|
+
skipped,
|
|
7960
|
+
warnings,
|
|
7961
|
+
errors: [`Failed to parse shard component: ${err instanceof Error ? err.message : String(err)}`],
|
|
7962
|
+
duration_ms: performance.now() - start
|
|
7963
|
+
};
|
|
7964
|
+
}
|
|
7300
7965
|
const knownFiles = /* @__PURE__ */ new Set([
|
|
7301
7966
|
"manifest.json",
|
|
7302
7967
|
"notes.jsonl",
|
|
@@ -7305,7 +7970,6 @@ async function importShard(db, data, options) {
|
|
|
7305
7970
|
"links.jsonl",
|
|
7306
7971
|
"embedding_sets.json",
|
|
7307
7972
|
"embedding_set_members.jsonl",
|
|
7308
|
-
"embedding_configs.json",
|
|
7309
7973
|
"embeddings.jsonl",
|
|
7310
7974
|
"templates.json",
|
|
7311
7975
|
"skos_schemes.json",
|
|
@@ -7323,15 +7987,36 @@ async function importShard(db, data, options) {
|
|
|
7323
7987
|
warnings.push(`Unknown component skipped: ${filename}`);
|
|
7324
7988
|
}
|
|
7325
7989
|
}
|
|
7326
|
-
if (files.has("templates.json")) {
|
|
7327
|
-
warnings.push("templates.json skipped (not supported in browser)");
|
|
7328
|
-
}
|
|
7329
7990
|
const conflictClause = strategy === "skip" ? "ON CONFLICT DO NOTHING" : "";
|
|
7330
7991
|
try {
|
|
7331
7992
|
await db.transaction(async (tx) => {
|
|
7993
|
+
const preexistingEmbeddingMembers = /* @__PURE__ */ new Set();
|
|
7994
|
+
if (strategy === "skip") {
|
|
7995
|
+
for (const member of parsedEmbMembers) {
|
|
7996
|
+
const existing = await tx.query(
|
|
7997
|
+
"SELECT 1 FROM embedding_set_member WHERE embedding_set_id = $1 AND note_id = $2",
|
|
7998
|
+
[member.embedding_set_id, member.note_id]
|
|
7999
|
+
);
|
|
8000
|
+
if (existing.rows.length > 0) {
|
|
8001
|
+
preexistingEmbeddingMembers.add(`${member.embedding_set_id}\0${member.note_id}`);
|
|
8002
|
+
}
|
|
8003
|
+
}
|
|
8004
|
+
}
|
|
8005
|
+
const skipExisting = async (key, sql, params) => {
|
|
8006
|
+
if (strategy !== "skip") return false;
|
|
8007
|
+
const existing = await tx.query(sql, params);
|
|
8008
|
+
if (existing.rows.length === 0) return false;
|
|
8009
|
+
skipped[key] = (skipped[key] ?? 0) + 1;
|
|
8010
|
+
return true;
|
|
8011
|
+
};
|
|
7332
8012
|
report?.({ phase: "collections", done: 0, total: parsedCollections.length });
|
|
7333
8013
|
for (const [index, shardCol] of parsedCollections.entries()) {
|
|
7334
8014
|
const col = collectionFromShard(shardCol);
|
|
8015
|
+
if (await skipExisting("collections", "SELECT 1 FROM collection WHERE id = $1", [col.id])) {
|
|
8016
|
+
report?.({ phase: "collections", done: index + 1, total: parsedCollections.length });
|
|
8017
|
+
await maybeYield2(index + 1, batchSize);
|
|
8018
|
+
continue;
|
|
8019
|
+
}
|
|
7335
8020
|
if (strategy === "replace") {
|
|
7336
8021
|
await tx.query(
|
|
7337
8022
|
`INSERT INTO collection (id, name, description, parent_id, created_at)
|
|
@@ -7350,9 +8035,38 @@ async function importShard(db, data, options) {
|
|
|
7350
8035
|
report?.({ phase: "collections", done: index + 1, total: parsedCollections.length });
|
|
7351
8036
|
await maybeYield2(index + 1, batchSize);
|
|
7352
8037
|
}
|
|
8038
|
+
report?.({ phase: "templates", done: 0, total: parsedTemplates.length });
|
|
8039
|
+
for (const [index, template] of parsedTemplates.entries()) {
|
|
8040
|
+
await tx.query(
|
|
8041
|
+
`INSERT INTO template (
|
|
8042
|
+
id, name, description, content, format, default_tags, collection_id, created_at, updated_at
|
|
8043
|
+
)
|
|
8044
|
+
VALUES ($1, $2, $3, $4, $5, $6::jsonb, $7, $8, $9)
|
|
8045
|
+
${strategy === "replace" ? "ON CONFLICT (id) DO UPDATE SET name = $2, description = $3, content = $4, format = $5, default_tags = $6::jsonb, collection_id = $7, created_at = $8, updated_at = $9" : conflictClause}`,
|
|
8046
|
+
[
|
|
8047
|
+
template.id,
|
|
8048
|
+
template.name,
|
|
8049
|
+
template.description,
|
|
8050
|
+
template.content,
|
|
8051
|
+
template.format,
|
|
8052
|
+
JSON.stringify(template.default_tags),
|
|
8053
|
+
template.collection_id,
|
|
8054
|
+
template.created_at,
|
|
8055
|
+
template.updated_at
|
|
8056
|
+
]
|
|
8057
|
+
);
|
|
8058
|
+
counts.templates++;
|
|
8059
|
+
report?.({ phase: "templates", done: index + 1, total: parsedTemplates.length });
|
|
8060
|
+
await maybeYield2(index + 1, batchSize);
|
|
8061
|
+
}
|
|
7353
8062
|
report?.({ phase: "notes", done: 0, total: parsedNotes.length });
|
|
7354
8063
|
for (const [index, shardNote] of parsedNotes.entries()) {
|
|
7355
8064
|
const note = noteFromShard(shardNote);
|
|
8065
|
+
if (await skipExisting("notes", "SELECT 1 FROM note WHERE id = $1", [note.id])) {
|
|
8066
|
+
report?.({ phase: "notes", done: index + 1, total: parsedNotes.length });
|
|
8067
|
+
await maybeYield2(index + 1, batchSize);
|
|
8068
|
+
continue;
|
|
8069
|
+
}
|
|
7356
8070
|
const contentHash = computeHash(new TextEncoder().encode(note.original_content));
|
|
7357
8071
|
if (strategy === "replace") {
|
|
7358
8072
|
await tx.query(
|
|
@@ -7427,23 +8141,98 @@ async function importShard(db, data, options) {
|
|
|
7427
8141
|
[generateId(), note.id, tag]
|
|
7428
8142
|
);
|
|
7429
8143
|
}
|
|
7430
|
-
if (note.
|
|
7431
|
-
|
|
7432
|
-
|
|
8144
|
+
if (note.collection_id) {
|
|
8145
|
+
await tx.query(
|
|
8146
|
+
`INSERT INTO collection_note (collection_id, note_id)
|
|
8147
|
+
VALUES ($1, $2)
|
|
8148
|
+
ON CONFLICT (collection_id, note_id) DO NOTHING`,
|
|
8149
|
+
[note.collection_id, note.id]
|
|
8150
|
+
);
|
|
8151
|
+
}
|
|
8152
|
+
if (note.attachments?.length) {
|
|
8153
|
+
notesWithImportedAttachmentReferences++;
|
|
8154
|
+
for (let position = 0; position < note.attachments.length; position += 1) {
|
|
8155
|
+
const projection = note.attachments[position];
|
|
8156
|
+
const ref = projection.attachment;
|
|
8157
|
+
const existingBlob = await tx.query(
|
|
8158
|
+
`SELECT id FROM attachment_blob WHERE content_hash = $1`,
|
|
8159
|
+
[ref.checksum]
|
|
8160
|
+
);
|
|
8161
|
+
const blobId = existingBlob.rows[0]?.id ?? generateId();
|
|
8162
|
+
if (!existingBlob.rows.length) {
|
|
8163
|
+
await tx.query(
|
|
8164
|
+
`INSERT INTO attachment_blob (id, content_hash, size_bytes, storage_path)
|
|
8165
|
+
VALUES ($1, $2, $3, $4)
|
|
8166
|
+
ON CONFLICT (content_hash) DO NOTHING`,
|
|
8167
|
+
// storage_path stays NULL: the browser edition addresses blobs
|
|
8168
|
+
// by content_hash via the BlobStore, not a filesystem key, and
|
|
8169
|
+
// `ref.path` is the display filename (never a storage locator).
|
|
8170
|
+
[blobId, ref.checksum, ref.bytes, null]
|
|
8171
|
+
);
|
|
8172
|
+
}
|
|
8173
|
+
const filename = ref.path.split("/").filter(Boolean).pop() ?? ref.path;
|
|
8174
|
+
if (strategy === "replace") {
|
|
8175
|
+
await tx.query(
|
|
8176
|
+
`INSERT INTO attachment (id, note_id, blob_id, filename, mime_type, extracted_text, position)
|
|
8177
|
+
VALUES ($1, $2, $3, $4, $5, $6, $7)
|
|
8178
|
+
ON CONFLICT (id) DO UPDATE SET
|
|
8179
|
+
note_id = $2,
|
|
8180
|
+
blob_id = $3,
|
|
8181
|
+
filename = $4,
|
|
8182
|
+
mime_type = $5,
|
|
8183
|
+
extracted_text = $6,
|
|
8184
|
+
position = $7,
|
|
8185
|
+
deleted_at = NULL`,
|
|
8186
|
+
[ref.id, note.id, blobId, filename, ref.mime, projection.extracted_text, position]
|
|
8187
|
+
);
|
|
8188
|
+
} else {
|
|
8189
|
+
await tx.query(
|
|
8190
|
+
`INSERT INTO attachment (id, note_id, blob_id, filename, mime_type, extracted_text, position)
|
|
8191
|
+
VALUES ($1, $2, $3, $4, $5, $6, $7) ${conflictClause}`,
|
|
8192
|
+
[ref.id, note.id, blobId, filename, ref.mime, projection.extracted_text, position]
|
|
8193
|
+
);
|
|
8194
|
+
}
|
|
8195
|
+
let hydrated = false;
|
|
8196
|
+
if (sidecarBlobs) {
|
|
8197
|
+
if (blobsToHydrate.has(ref.checksum)) {
|
|
8198
|
+
hydrated = true;
|
|
8199
|
+
} else {
|
|
8200
|
+
const bytes = sidecarBlobs.get(blobChecksumToHex(ref.checksum));
|
|
8201
|
+
if (bytes) {
|
|
8202
|
+
if (computeBlobHash(bytes) === ref.checksum) {
|
|
8203
|
+
blobsToHydrate.set(ref.checksum, bytes);
|
|
8204
|
+
hydrated = true;
|
|
8205
|
+
} else {
|
|
8206
|
+
warnings.push(
|
|
8207
|
+
`Sidecar blob for attachment ${ref.id} failed BLAKE3 integrity check (expected ${ref.checksum}); imported as reference-only.`
|
|
8208
|
+
);
|
|
8209
|
+
}
|
|
8210
|
+
}
|
|
8211
|
+
}
|
|
8212
|
+
}
|
|
8213
|
+
if (!hydrated) {
|
|
8214
|
+
importedAttachmentReferenceCount++;
|
|
8215
|
+
}
|
|
8216
|
+
}
|
|
7433
8217
|
}
|
|
7434
8218
|
counts.notes++;
|
|
7435
8219
|
report?.({ phase: "notes", done: index + 1, total: parsedNotes.length });
|
|
7436
8220
|
await maybeYield2(index + 1, batchSize);
|
|
7437
8221
|
}
|
|
7438
|
-
if (
|
|
8222
|
+
if (importedAttachmentReferenceCount > 0) {
|
|
7439
8223
|
warnings.push(
|
|
7440
|
-
`${
|
|
8224
|
+
`${importedAttachmentReferenceCount} attachment reference(s) across ${notesWithImportedAttachmentReferences} note(s) were imported as metadata only: these shard records carry extracted text plus attachment metadata but no matching byte-sidecar entry, so their BlobStore bytes are unavailable (getBlob() returns null). Export a self-contained shard (\`includeBlobs\` + a \`blobStore\`) and import with a \`blobStore\` to hydrate bytes. Tracking: #271 / server #1046.`
|
|
7441
8225
|
);
|
|
7442
8226
|
}
|
|
7443
8227
|
const totalSkos = parsedSkosSchemes.length + parsedSkosConcepts.length + parsedSkosRelations.length + parsedNoteSkosTags.length;
|
|
7444
8228
|
let doneSkos = 0;
|
|
7445
8229
|
report?.({ phase: "skos", done: doneSkos, total: totalSkos });
|
|
7446
8230
|
for (const scheme of parsedSkosSchemes) {
|
|
8231
|
+
if (await skipExisting("skos_schemes", "SELECT 1 FROM skos_scheme WHERE id = $1", [scheme.id])) {
|
|
8232
|
+
report?.({ phase: "skos", done: ++doneSkos, total: totalSkos });
|
|
8233
|
+
await maybeYield2(doneSkos, batchSize);
|
|
8234
|
+
continue;
|
|
8235
|
+
}
|
|
7447
8236
|
if (strategy === "replace") {
|
|
7448
8237
|
await tx.query(
|
|
7449
8238
|
`INSERT INTO skos_scheme (id, title, description, created_at, updated_at)
|
|
@@ -7463,6 +8252,11 @@ async function importShard(db, data, options) {
|
|
|
7463
8252
|
await maybeYield2(doneSkos, batchSize);
|
|
7464
8253
|
}
|
|
7465
8254
|
for (const concept of parsedSkosConcepts) {
|
|
8255
|
+
if (await skipExisting("skos_concepts", "SELECT 1 FROM skos_concept WHERE id = $1", [concept.id])) {
|
|
8256
|
+
report?.({ phase: "skos", done: ++doneSkos, total: totalSkos });
|
|
8257
|
+
await maybeYield2(doneSkos, batchSize);
|
|
8258
|
+
continue;
|
|
8259
|
+
}
|
|
7466
8260
|
const altLabels = JSON.stringify(concept.alt_labels ?? []);
|
|
7467
8261
|
if (strategy === "replace") {
|
|
7468
8262
|
await tx.query(
|
|
@@ -7485,7 +8279,34 @@ async function importShard(db, data, options) {
|
|
|
7485
8279
|
report?.({ phase: "links", done: 0, total: parsedLinks.length });
|
|
7486
8280
|
for (const [index, shardLink] of parsedLinks.entries()) {
|
|
7487
8281
|
const link = linkFromShard(shardLink);
|
|
7488
|
-
if (
|
|
8282
|
+
if (!link.target_note_id) {
|
|
8283
|
+
if (!link.to_url) {
|
|
8284
|
+
skipped.links = (skipped.links ?? 0) + 1;
|
|
8285
|
+
warnings.push(`Shard link skipped: ${link.id} has neither to_note_id nor to_url.`);
|
|
8286
|
+
report?.({ phase: "links", done: index + 1, total: parsedLinks.length });
|
|
8287
|
+
await maybeYield2(index + 1, batchSize);
|
|
8288
|
+
continue;
|
|
8289
|
+
}
|
|
8290
|
+
const metadata = link.metadata == null ? null : JSON.stringify(link.metadata);
|
|
8291
|
+
await tx.query(
|
|
8292
|
+
`INSERT INTO link_url_target (
|
|
8293
|
+
id, source_note_id, to_url, link_type, confidence, metadata_json, created_at
|
|
8294
|
+
)
|
|
8295
|
+
VALUES ($1, $2, $3, $4, $5, $6::jsonb, $7)
|
|
8296
|
+
${strategy === "replace" ? "ON CONFLICT (id) DO UPDATE SET source_note_id = $2, to_url = $3, link_type = $4, confidence = $5, metadata_json = $6::jsonb, created_at = $7" : conflictClause}`,
|
|
8297
|
+
[link.id, link.source_note_id, link.to_url, link.link_type, link.confidence, metadata, link.created_at]
|
|
8298
|
+
);
|
|
8299
|
+
counts.links++;
|
|
8300
|
+
report?.({ phase: "links", done: index + 1, total: parsedLinks.length });
|
|
8301
|
+
await maybeYield2(index + 1, batchSize);
|
|
8302
|
+
continue;
|
|
8303
|
+
}
|
|
8304
|
+
if (await skipExisting("links", "SELECT 1 FROM link WHERE id = $1", [link.id])) {
|
|
8305
|
+
report?.({ phase: "links", done: index + 1, total: parsedLinks.length });
|
|
8306
|
+
await maybeYield2(index + 1, batchSize);
|
|
8307
|
+
continue;
|
|
8308
|
+
}
|
|
8309
|
+
if (strategy === "replace") {
|
|
7489
8310
|
await tx.query(
|
|
7490
8311
|
`INSERT INTO link (id, source_note_id, target_note_id, link_type, confidence, created_at)
|
|
7491
8312
|
VALUES ($1, $2, $3, $4, $5, $6)
|
|
@@ -7504,6 +8325,11 @@ async function importShard(db, data, options) {
|
|
|
7504
8325
|
await maybeYield2(index + 1, batchSize);
|
|
7505
8326
|
}
|
|
7506
8327
|
for (const relation of parsedSkosRelations) {
|
|
8328
|
+
if (await skipExisting("skos_relations", "SELECT 1 FROM skos_concept_relation WHERE id = $1", [relation.id])) {
|
|
8329
|
+
report?.({ phase: "skos", done: ++doneSkos, total: totalSkos });
|
|
8330
|
+
await maybeYield2(doneSkos, batchSize);
|
|
8331
|
+
continue;
|
|
8332
|
+
}
|
|
7507
8333
|
if (strategy === "replace") {
|
|
7508
8334
|
await tx.query(
|
|
7509
8335
|
`INSERT INTO skos_concept_relation (id, source_concept_id, target_concept_id, relation_type, created_at)
|
|
@@ -7523,6 +8349,11 @@ async function importShard(db, data, options) {
|
|
|
7523
8349
|
await maybeYield2(doneSkos, batchSize);
|
|
7524
8350
|
}
|
|
7525
8351
|
for (const tag of parsedNoteSkosTags) {
|
|
8352
|
+
if (await skipExisting("note_skos_tags", "SELECT 1 FROM note_skos_tag WHERE note_id = $1 AND concept_id = $2", [tag.note_id, tag.concept_id])) {
|
|
8353
|
+
report?.({ phase: "skos", done: ++doneSkos, total: totalSkos });
|
|
8354
|
+
await maybeYield2(doneSkos, batchSize);
|
|
8355
|
+
continue;
|
|
8356
|
+
}
|
|
7526
8357
|
await tx.query(
|
|
7527
8358
|
`INSERT INTO note_skos_tag (id, note_id, concept_id, created_at)
|
|
7528
8359
|
VALUES ($1, $2, $3, $4)
|
|
@@ -7535,6 +8366,11 @@ async function importShard(db, data, options) {
|
|
|
7535
8366
|
}
|
|
7536
8367
|
report?.({ phase: "provenance", done: 0, total: parsedProvenanceEdges.length });
|
|
7537
8368
|
for (const [index, edge] of parsedProvenanceEdges.entries()) {
|
|
8369
|
+
if (await skipExisting("provenance_edges", "SELECT 1 FROM provenance_edge WHERE id = $1", [edge.id])) {
|
|
8370
|
+
report?.({ phase: "provenance", done: index + 1, total: parsedProvenanceEdges.length });
|
|
8371
|
+
await maybeYield2(index + 1, batchSize);
|
|
8372
|
+
continue;
|
|
8373
|
+
}
|
|
7538
8374
|
const attributes = edge.attributes === null ? null : JSON.stringify(edge.attributes);
|
|
7539
8375
|
if (strategy === "replace") {
|
|
7540
8376
|
await tx.query(
|
|
@@ -7554,27 +8390,59 @@ async function importShard(db, data, options) {
|
|
|
7554
8390
|
report?.({ phase: "provenance", done: index + 1, total: parsedProvenanceEdges.length });
|
|
7555
8391
|
await maybeYield2(index + 1, batchSize);
|
|
7556
8392
|
}
|
|
8393
|
+
report?.({ phase: "embedding_configs", done: 0, total: parsedEmbConfigs.length });
|
|
8394
|
+
for (const [index, config] of parsedEmbConfigs.entries()) {
|
|
8395
|
+
await tx.query(
|
|
8396
|
+
`INSERT INTO embedding_config (
|
|
8397
|
+
id, name, description, model, dimension, chunk_size, chunk_overlap, is_default
|
|
8398
|
+
)
|
|
8399
|
+
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
|
|
8400
|
+
${strategy === "replace" ? "ON CONFLICT (id) DO UPDATE SET name = $2, description = $3, model = $4, dimension = $5, chunk_size = $6, chunk_overlap = $7, is_default = $8" : conflictClause}`,
|
|
8401
|
+
[
|
|
8402
|
+
config.id,
|
|
8403
|
+
config.name,
|
|
8404
|
+
config.description,
|
|
8405
|
+
config.model,
|
|
8406
|
+
config.dimension,
|
|
8407
|
+
config.chunk_size,
|
|
8408
|
+
config.chunk_overlap,
|
|
8409
|
+
config.is_default
|
|
8410
|
+
]
|
|
8411
|
+
);
|
|
8412
|
+
counts.embedding_configs++;
|
|
8413
|
+
report?.({ phase: "embedding_configs", done: index + 1, total: parsedEmbConfigs.length });
|
|
8414
|
+
await maybeYield2(index + 1, batchSize);
|
|
8415
|
+
}
|
|
7557
8416
|
report?.({ phase: "embedding_sets", done: 0, total: parsedEmbSets.length });
|
|
7558
8417
|
for (const [index, shardSet] of parsedEmbSets.entries()) {
|
|
7559
|
-
const set = embeddingSetFromShard(shardSet);
|
|
8418
|
+
const set = embeddingSetFromShard(shardSet, manifest.created_at);
|
|
8419
|
+
if (await skipExisting("embedding_sets", "SELECT 1 FROM embedding_set WHERE id = $1", [set.id])) {
|
|
8420
|
+
report?.({ phase: "embedding_sets", done: index + 1, total: parsedEmbSets.length });
|
|
8421
|
+
await maybeYield2(index + 1, batchSize);
|
|
8422
|
+
continue;
|
|
8423
|
+
}
|
|
7560
8424
|
if (strategy === "replace") {
|
|
7561
8425
|
await tx.query(
|
|
7562
8426
|
`INSERT INTO embedding_set (
|
|
7563
|
-
id, name,
|
|
8427
|
+
id, name, slug, description, purpose, document_count, embedding_count, is_system, keywords_json,
|
|
8428
|
+
model_name, dimensions, kind, mode, truncate_dimension,
|
|
7564
8429
|
criteria_json, source_json, compatibility_json, materialization_json, freshness_json, created_at, updated_at
|
|
7565
|
-
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::jsonb, $10::jsonb, $
|
|
7566
|
-
ON CONFLICT (id) DO UPDATE SET name = $2,
|
|
7567
|
-
|
|
7568
|
-
|
|
7569
|
-
|
|
8430
|
+
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::jsonb, $10, $11, $12, $13, $14, $15::jsonb, $16::jsonb, $17::jsonb, $18::jsonb, $19::jsonb, $20, COALESCE($21::timestamptz, $20::timestamptz))
|
|
8431
|
+
ON CONFLICT (id) DO UPDATE SET name = $2, slug = $3, description = $4, purpose = $5,
|
|
8432
|
+
document_count = $6, embedding_count = $7, is_system = $8, keywords_json = $9::jsonb,
|
|
8433
|
+
model_name = $10, dimensions = $11, kind = $12, mode = $13, truncate_dimension = $14,
|
|
8434
|
+
criteria_json = $15::jsonb, source_json = $16::jsonb, compatibility_json = $17::jsonb,
|
|
8435
|
+
materialization_json = $18::jsonb, freshness_json = $19::jsonb, updated_at = COALESCE($21::timestamptz, $20::timestamptz)`,
|
|
8436
|
+
[set.id, set.name, set.slug, set.description, set.purpose, set.document_count, set.embedding_count, set.is_system, set.keywords_json, set.model_name, set.dimensions, set.kind, set.mode, set.truncate_dimension, set.criteria_json, set.source_json, set.compatibility_json, set.materialization_json, set.freshness_json, set.created_at, set.updated_at]
|
|
7570
8437
|
);
|
|
7571
8438
|
} else {
|
|
7572
8439
|
await tx.query(
|
|
7573
8440
|
`INSERT INTO embedding_set (
|
|
7574
|
-
id, name,
|
|
8441
|
+
id, name, slug, description, purpose, document_count, embedding_count, is_system, keywords_json,
|
|
8442
|
+
model_name, dimensions, kind, mode, truncate_dimension,
|
|
7575
8443
|
criteria_json, source_json, compatibility_json, materialization_json, freshness_json, created_at, updated_at
|
|
7576
|
-
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::jsonb, $10::jsonb, $
|
|
7577
|
-
[set.id, set.name, set.purpose, set.model_name, set.dimensions, set.kind, set.mode, set.truncate_dimension, set.criteria_json, set.source_json, set.compatibility_json, set.materialization_json, set.freshness_json, set.created_at, set.updated_at]
|
|
8444
|
+
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::jsonb, $10, $11, $12, $13, $14, $15::jsonb, $16::jsonb, $17::jsonb, $18::jsonb, $19::jsonb, $20, COALESCE($21::timestamptz, $20::timestamptz)) ${conflictClause}`,
|
|
8445
|
+
[set.id, set.name, set.slug, set.description, set.purpose, set.document_count, set.embedding_count, set.is_system, set.keywords_json, set.model_name, set.dimensions, set.kind, set.mode, set.truncate_dimension, set.criteria_json, set.source_json, set.compatibility_json, set.materialization_json, set.freshness_json, set.created_at, set.updated_at]
|
|
7578
8446
|
);
|
|
7579
8447
|
}
|
|
7580
8448
|
counts.embedding_sets++;
|
|
@@ -7584,20 +8452,38 @@ async function importShard(db, data, options) {
|
|
|
7584
8452
|
report?.({ phase: "embeddings", done: 0, total: parsedEmbeddings.length });
|
|
7585
8453
|
for (const [index, shardEmb] of parsedEmbeddings.entries()) {
|
|
7586
8454
|
const emb = embeddingFromShard(shardEmb);
|
|
8455
|
+
const embeddingSetId = emb.embedding_set_id ?? await resolveEmbeddingSetIdForServerEmbedding(tx, emb.model, emb.vector);
|
|
8456
|
+
if (await skipExisting("embeddings", "SELECT 1 FROM embedding WHERE id = $1", [emb.id])) {
|
|
8457
|
+
report?.({ phase: "embeddings", done: index + 1, total: parsedEmbeddings.length });
|
|
8458
|
+
await maybeYield2(index + 1, batchSize);
|
|
8459
|
+
continue;
|
|
8460
|
+
}
|
|
7587
8461
|
if (strategy === "replace") {
|
|
7588
8462
|
await tx.query(
|
|
7589
|
-
`INSERT INTO embedding (id, note_id, embedding_set_id, vector, created_at)
|
|
7590
|
-
VALUES ($1, $2, $3, $4, $5)
|
|
7591
|
-
ON CONFLICT (id) DO UPDATE SET vector = $
|
|
7592
|
-
[emb.id, emb.note_id, emb.
|
|
8463
|
+
`INSERT INTO embedding (id, note_id, embedding_set_id, chunk_index, text, vector, model, created_at)
|
|
8464
|
+
VALUES ($1, $2, $3, $4, $5, $6, $7, COALESCE($8::timestamptz, now()))
|
|
8465
|
+
ON CONFLICT (id) DO UPDATE SET embedding_set_id = $3, chunk_index = $4, text = $5, vector = $6, model = $7`,
|
|
8466
|
+
[emb.id, emb.note_id, embeddingSetId, emb.chunk_index, emb.text, emb.vector, emb.model, emb.created_at]
|
|
7593
8467
|
);
|
|
7594
8468
|
} else {
|
|
7595
8469
|
await tx.query(
|
|
7596
|
-
`INSERT INTO embedding (id, note_id, embedding_set_id, vector, created_at)
|
|
7597
|
-
VALUES ($1, $2, $3, $4, $5) ${conflictClause}`,
|
|
7598
|
-
[emb.id, emb.note_id, emb.
|
|
8470
|
+
`INSERT INTO embedding (id, note_id, embedding_set_id, chunk_index, text, vector, model, created_at)
|
|
8471
|
+
VALUES ($1, $2, $3, $4, $5, $6, $7, COALESCE($8::timestamptz, now())) ${conflictClause}`,
|
|
8472
|
+
[emb.id, emb.note_id, embeddingSetId, emb.chunk_index, emb.text, emb.vector, emb.model, emb.created_at]
|
|
7599
8473
|
);
|
|
7600
8474
|
}
|
|
8475
|
+
await tx.query(
|
|
8476
|
+
`INSERT INTO embedding_set_member (
|
|
8477
|
+
embedding_set_id, note_id, embedding_id, membership_type, added_at, added_by
|
|
8478
|
+
) VALUES ($1, $2, $3, 'materialized', COALESCE($4::timestamptz, now()), 'shard-import')
|
|
8479
|
+
ON CONFLICT (embedding_set_id, note_id) DO UPDATE SET
|
|
8480
|
+
embedding_id = EXCLUDED.embedding_id,
|
|
8481
|
+
membership_type = CASE
|
|
8482
|
+
WHEN embedding_set_member.membership_type = 'auto' THEN 'materialized'
|
|
8483
|
+
ELSE embedding_set_member.membership_type
|
|
8484
|
+
END`,
|
|
8485
|
+
[embeddingSetId, emb.note_id, emb.id, emb.created_at]
|
|
8486
|
+
);
|
|
7601
8487
|
counts.embeddings++;
|
|
7602
8488
|
report?.({ phase: "embeddings", done: index + 1, total: parsedEmbeddings.length });
|
|
7603
8489
|
await maybeYield2(index + 1, batchSize);
|
|
@@ -7606,6 +8492,11 @@ async function importShard(db, data, options) {
|
|
|
7606
8492
|
let doneGraph = 0;
|
|
7607
8493
|
report?.({ phase: "graph", done: doneGraph, total: totalGraph });
|
|
7608
8494
|
for (const source of parsedGraphSources) {
|
|
8495
|
+
if (await skipExisting("graph_sources", "SELECT 1 FROM graph_source WHERE id = $1", [source.id])) {
|
|
8496
|
+
report?.({ phase: "graph", done: ++doneGraph, total: totalGraph });
|
|
8497
|
+
await maybeYield2(doneGraph, batchSize);
|
|
8498
|
+
continue;
|
|
8499
|
+
}
|
|
7609
8500
|
const parameters = source.parameters == null ? null : JSON.stringify(source.parameters);
|
|
7610
8501
|
const freshness = JSON.stringify({ ...source.freshness ?? {}, status: "unknown" });
|
|
7611
8502
|
await tx.query(
|
|
@@ -7621,6 +8512,15 @@ async function importShard(db, data, options) {
|
|
|
7621
8512
|
await maybeYield2(doneGraph, batchSize);
|
|
7622
8513
|
}
|
|
7623
8514
|
for (const edge of parsedGraphEdges) {
|
|
8515
|
+
if (await skipExisting(
|
|
8516
|
+
"graph_edges",
|
|
8517
|
+
"SELECT 1 FROM graph_edge_artifact WHERE graph_source_id = $1 AND from_note_id = $2 AND to_note_id = $3 AND kind = $4",
|
|
8518
|
+
[edge.graph_source_id, edge.from_note_id, edge.to_note_id, edge.kind]
|
|
8519
|
+
)) {
|
|
8520
|
+
report?.({ phase: "graph", done: ++doneGraph, total: totalGraph });
|
|
8521
|
+
await maybeYield2(doneGraph, batchSize);
|
|
8522
|
+
continue;
|
|
8523
|
+
}
|
|
7624
8524
|
const metadata = edge.metadata == null ? null : JSON.stringify(edge.metadata);
|
|
7625
8525
|
await tx.query(
|
|
7626
8526
|
`INSERT INTO graph_edge_artifact (graph_source_id, from_note_id, to_note_id, weight, kind, rank, metadata_json)
|
|
@@ -7638,16 +8538,22 @@ async function importShard(db, data, options) {
|
|
|
7638
8538
|
for (const set of parsedCommunitySets) {
|
|
7639
8539
|
const parameters = set.parameters == null ? null : JSON.stringify(set.parameters);
|
|
7640
8540
|
const freshness = JSON.stringify({ ...set.freshness ?? {}, status: "unknown" });
|
|
7641
|
-
await
|
|
8541
|
+
const skippedSet = await skipExisting("community_sets", "SELECT 1 FROM community_set WHERE id = $1", [set.id]);
|
|
8542
|
+
if (!skippedSet) await tx.query(
|
|
7642
8543
|
`INSERT INTO community_set (id, graph_source_id, name, source_type, algorithm, parameters_json, input_hash, freshness_json, created_at)
|
|
7643
8544
|
VALUES ($1, $2, $3, $4, $5, $6::jsonb, $7, $8::jsonb, $9)
|
|
7644
8545
|
${strategy === "replace" ? "ON CONFLICT (id) DO UPDATE SET graph_source_id = $2, name = $3, source_type = $4, algorithm = $5, parameters_json = $6::jsonb, input_hash = $7, freshness_json = $8::jsonb, created_at = $9" : conflictClause}`,
|
|
7645
8546
|
[set.id, set.graph_source_id, set.name, set.source_type, set.algorithm ?? null, parameters, set.input_hash, freshness, set.created_at]
|
|
7646
8547
|
);
|
|
7647
|
-
counts.community_sets++;
|
|
8548
|
+
if (!skippedSet) counts.community_sets++;
|
|
7648
8549
|
report?.({ phase: "communities", done: ++doneCommunities, total: totalCommunities });
|
|
7649
8550
|
await maybeYield2(doneCommunities, batchSize);
|
|
7650
8551
|
for (const community of set.communities ?? []) {
|
|
8552
|
+
if (await skipExisting("communities", "SELECT 1 FROM community WHERE community_set_id = $1 AND id = $2", [set.id, community.id])) {
|
|
8553
|
+
report?.({ phase: "communities", done: ++doneCommunities, total: totalCommunities });
|
|
8554
|
+
await maybeYield2(doneCommunities, batchSize);
|
|
8555
|
+
continue;
|
|
8556
|
+
}
|
|
7651
8557
|
const metadata = community.metadata == null ? null : JSON.stringify(community.metadata);
|
|
7652
8558
|
await tx.query(
|
|
7653
8559
|
`INSERT INTO community (community_set_id, id, label, rank, size, confidence, representative_note_ids, metadata_json)
|
|
@@ -7661,6 +8567,11 @@ async function importShard(db, data, options) {
|
|
|
7661
8567
|
}
|
|
7662
8568
|
}
|
|
7663
8569
|
for (const assignment of parsedCommunityAssignments) {
|
|
8570
|
+
if (await skipExisting("community_assignments", "SELECT 1 FROM community_assignment WHERE community_set_id = $1 AND note_id = $2", [assignment.community_set_id, assignment.note_id])) {
|
|
8571
|
+
report?.({ phase: "communities", done: ++doneCommunities, total: totalCommunities });
|
|
8572
|
+
await maybeYield2(doneCommunities, batchSize);
|
|
8573
|
+
continue;
|
|
8574
|
+
}
|
|
7664
8575
|
const metadata = assignment.metadata == null ? null : JSON.stringify(assignment.metadata);
|
|
7665
8576
|
await tx.query(
|
|
7666
8577
|
`INSERT INTO community_assignment (community_set_id, community_id, note_id, confidence, source_type, metadata_json)
|
|
@@ -7674,10 +8585,38 @@ async function importShard(db, data, options) {
|
|
|
7674
8585
|
}
|
|
7675
8586
|
report?.({ phase: "embedding_set_members", done: 0, total: parsedEmbMembers.length });
|
|
7676
8587
|
for (const [index, member] of parsedEmbMembers.entries()) {
|
|
8588
|
+
if (strategy === "skip" && preexistingEmbeddingMembers.has(`${member.embedding_set_id}\0${member.note_id}`)) {
|
|
8589
|
+
skipped.embedding_set_members = (skipped.embedding_set_members ?? 0) + 1;
|
|
8590
|
+
report?.({ phase: "embedding_set_members", done: index + 1, total: parsedEmbMembers.length });
|
|
8591
|
+
await maybeYield2(index + 1, batchSize);
|
|
8592
|
+
continue;
|
|
8593
|
+
}
|
|
8594
|
+
let embeddingId = member.embedding_id ?? null;
|
|
8595
|
+
if (!embeddingId) {
|
|
8596
|
+
const resolvedEmbedding = await tx.query(
|
|
8597
|
+
`SELECT id FROM embedding WHERE embedding_set_id = $1 AND note_id = $2 ORDER BY created_at DESC LIMIT 1`,
|
|
8598
|
+
[member.embedding_set_id, member.note_id]
|
|
8599
|
+
);
|
|
8600
|
+
embeddingId = resolvedEmbedding.rows[0]?.id ?? null;
|
|
8601
|
+
}
|
|
7677
8602
|
await tx.query(
|
|
7678
|
-
`INSERT INTO embedding_set_member (
|
|
7679
|
-
|
|
7680
|
-
|
|
8603
|
+
`INSERT INTO embedding_set_member (
|
|
8604
|
+
embedding_set_id, note_id, embedding_id, membership_type, added_at, added_by
|
|
8605
|
+
)
|
|
8606
|
+
VALUES ($1, $2, $3, $4, $5, $6)
|
|
8607
|
+
ON CONFLICT (embedding_set_id, note_id) DO UPDATE SET
|
|
8608
|
+
embedding_id = COALESCE(EXCLUDED.embedding_id, embedding_set_member.embedding_id),
|
|
8609
|
+
membership_type = EXCLUDED.membership_type,
|
|
8610
|
+
added_at = EXCLUDED.added_at,
|
|
8611
|
+
added_by = EXCLUDED.added_by`,
|
|
8612
|
+
[
|
|
8613
|
+
member.embedding_set_id,
|
|
8614
|
+
member.note_id,
|
|
8615
|
+
embeddingId,
|
|
8616
|
+
member.membership_type ?? "materialized",
|
|
8617
|
+
member.added_at ?? manifest.created_at,
|
|
8618
|
+
member.added_by ?? null
|
|
8619
|
+
]
|
|
7681
8620
|
);
|
|
7682
8621
|
counts.embedding_set_members++;
|
|
7683
8622
|
report?.({ phase: "embedding_set_members", done: index + 1, total: parsedEmbMembers.length });
|
|
@@ -7695,6 +8634,17 @@ async function importShard(db, data, options) {
|
|
|
7695
8634
|
duration_ms: performance.now() - start
|
|
7696
8635
|
};
|
|
7697
8636
|
}
|
|
8637
|
+
if (options?.blobStore && blobsToHydrate.size > 0) {
|
|
8638
|
+
try {
|
|
8639
|
+
for (const [hash, bytes] of blobsToHydrate) {
|
|
8640
|
+
await options.blobStore.write(hash, bytes);
|
|
8641
|
+
}
|
|
8642
|
+
} catch (err) {
|
|
8643
|
+
warnings.push(
|
|
8644
|
+
`Imported metadata successfully but failed to hydrate ${blobsToHydrate.size} attachment blob(s) into the BlobStore: ${err instanceof Error ? err.message : String(err)}.`
|
|
8645
|
+
);
|
|
8646
|
+
}
|
|
8647
|
+
}
|
|
7698
8648
|
return {
|
|
7699
8649
|
success: true,
|
|
7700
8650
|
counts,
|
|
@@ -7704,31 +8654,667 @@ async function importShard(db, data, options) {
|
|
|
7704
8654
|
duration_ms: performance.now() - start
|
|
7705
8655
|
};
|
|
7706
8656
|
}
|
|
7707
|
-
function
|
|
7708
|
-
|
|
7709
|
-
const
|
|
7710
|
-
|
|
8657
|
+
async function resolveEmbeddingSetIdForServerEmbedding(db, model, vector) {
|
|
8658
|
+
const dimension = vectorDimension(vector);
|
|
8659
|
+
const existing = await db.query(
|
|
8660
|
+
`SELECT id FROM embedding_set
|
|
8661
|
+
WHERE model_name = $1 AND dimensions = $2
|
|
8662
|
+
ORDER BY created_at, id
|
|
8663
|
+
LIMIT 1`,
|
|
8664
|
+
[model, dimension]
|
|
8665
|
+
);
|
|
8666
|
+
if (existing.rows[0]?.id) return existing.rows[0].id;
|
|
8667
|
+
const id = generateId();
|
|
8668
|
+
await db.query(
|
|
8669
|
+
`INSERT INTO embedding_set (
|
|
8670
|
+
id, name, slug, description, purpose, document_count, embedding_count,
|
|
8671
|
+
is_system, keywords_json, model_name, dimensions, kind, created_at, updated_at
|
|
8672
|
+
) VALUES ($1, $2, $3, NULL, NULL, 0, 0, false, '[]'::jsonb, $2, $4, 'physical', now(), now())`,
|
|
8673
|
+
[id, model, slugifyServerEmbeddingSet(model), dimension]
|
|
8674
|
+
);
|
|
8675
|
+
return id;
|
|
7711
8676
|
}
|
|
7712
|
-
function
|
|
7713
|
-
|
|
7714
|
-
|
|
8677
|
+
function vectorDimension(vector) {
|
|
8678
|
+
const inner = vector.replace(/^\[/, "").replace(/\]$/, "").trim();
|
|
8679
|
+
if (!inner) return 0;
|
|
8680
|
+
return inner.split(",").length;
|
|
8681
|
+
}
|
|
8682
|
+
function slugifyServerEmbeddingSet(value) {
|
|
8683
|
+
const slug = value.trim().toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "");
|
|
8684
|
+
return slug || "embedding-set";
|
|
8685
|
+
}
|
|
8686
|
+
|
|
8687
|
+
// schemas/knowledge-shard.schema.json
|
|
8688
|
+
var knowledge_shard_schema_default = {
|
|
8689
|
+
$schema: "https://json-schema.org/draft/2020-12/schema",
|
|
8690
|
+
$id: "https://fortemi.dev/schemas/knowledge-shard.schema.json",
|
|
8691
|
+
title: "Knowledge Shard (matric-shard) conformance schema",
|
|
8692
|
+
description: "Structural authority for the Knowledge Shard interchange contract shared between the Fortemi server (GET /api/v1/backup/knowledge-shard) and @fortemi/core. Covers the manifest, server-owned shard components, and React extension components tracked by issue #255.",
|
|
8693
|
+
type: "object",
|
|
8694
|
+
$defs: {
|
|
8695
|
+
isoDateTime: {
|
|
8696
|
+
type: "string",
|
|
8697
|
+
description: "ISO 8601 timestamp",
|
|
8698
|
+
minLength: 1
|
|
8699
|
+
},
|
|
8700
|
+
jsonObject: {
|
|
8701
|
+
type: "object",
|
|
8702
|
+
additionalProperties: true
|
|
8703
|
+
},
|
|
8704
|
+
nullableJsonObject: {
|
|
8705
|
+
type: ["object", "null"],
|
|
8706
|
+
additionalProperties: true
|
|
8707
|
+
},
|
|
8708
|
+
stringArray: {
|
|
8709
|
+
type: "array",
|
|
8710
|
+
items: { type: "string" }
|
|
8711
|
+
},
|
|
8712
|
+
manifest: {
|
|
8713
|
+
type: "object",
|
|
8714
|
+
additionalProperties: false,
|
|
8715
|
+
required: [
|
|
8716
|
+
"version",
|
|
8717
|
+
"matric_version",
|
|
8718
|
+
"format",
|
|
8719
|
+
"created_at",
|
|
8720
|
+
"components",
|
|
8721
|
+
"counts",
|
|
8722
|
+
"checksums",
|
|
8723
|
+
"min_reader_version"
|
|
8724
|
+
],
|
|
8725
|
+
properties: {
|
|
8726
|
+
version: { type: "string", minLength: 1 },
|
|
8727
|
+
matric_version: { type: "string", minLength: 1 },
|
|
8728
|
+
format: { const: "matric-shard" },
|
|
8729
|
+
created_at: { $ref: "#/$defs/isoDateTime" },
|
|
8730
|
+
components: {
|
|
8731
|
+
type: "array",
|
|
8732
|
+
items: {
|
|
8733
|
+
enum: [
|
|
8734
|
+
"notes",
|
|
8735
|
+
"collections",
|
|
8736
|
+
"tags",
|
|
8737
|
+
"templates",
|
|
8738
|
+
"links",
|
|
8739
|
+
"embedding_sets",
|
|
8740
|
+
"embedding_configs",
|
|
8741
|
+
"embedding_set_members",
|
|
8742
|
+
"embeddings",
|
|
8743
|
+
"skos_schemes",
|
|
8744
|
+
"skos_concepts",
|
|
8745
|
+
"skos_relations",
|
|
8746
|
+
"note_skos_tags",
|
|
8747
|
+
"provenance_edges",
|
|
8748
|
+
"community_assignments",
|
|
8749
|
+
"communities",
|
|
8750
|
+
"graph_edges",
|
|
8751
|
+
"graph_sources"
|
|
8752
|
+
]
|
|
8753
|
+
}
|
|
8754
|
+
},
|
|
8755
|
+
counts: {
|
|
8756
|
+
type: "object",
|
|
8757
|
+
additionalProperties: { type: "integer", minimum: 0 }
|
|
8758
|
+
},
|
|
8759
|
+
checksums: {
|
|
8760
|
+
type: "object",
|
|
8761
|
+
description: "filename -> sha256 hex",
|
|
8762
|
+
additionalProperties: { type: "string", pattern: "^[0-9a-f]{64}$" }
|
|
8763
|
+
},
|
|
8764
|
+
min_reader_version: { type: "string", minLength: 1 },
|
|
8765
|
+
migrated_from: {
|
|
8766
|
+
type: ["string", "null"],
|
|
8767
|
+
minLength: 1
|
|
8768
|
+
},
|
|
8769
|
+
migration_history: {
|
|
8770
|
+
type: "array",
|
|
8771
|
+
items: {
|
|
8772
|
+
type: "object",
|
|
8773
|
+
additionalProperties: false,
|
|
8774
|
+
required: [
|
|
8775
|
+
"from_version",
|
|
8776
|
+
"to_version",
|
|
8777
|
+
"migrated_at",
|
|
8778
|
+
"migrated_by",
|
|
8779
|
+
"changes"
|
|
8780
|
+
],
|
|
8781
|
+
properties: {
|
|
8782
|
+
from_version: { type: "string", minLength: 1 },
|
|
8783
|
+
to_version: { type: "string", minLength: 1 },
|
|
8784
|
+
migrated_at: { $ref: "#/$defs/isoDateTime" },
|
|
8785
|
+
migrated_by: { type: "string", minLength: 1 },
|
|
8786
|
+
changes: { $ref: "#/$defs/stringArray" }
|
|
8787
|
+
}
|
|
8788
|
+
}
|
|
8789
|
+
},
|
|
8790
|
+
layout: { type: "object" }
|
|
8791
|
+
}
|
|
8792
|
+
},
|
|
8793
|
+
attachmentReference: {
|
|
8794
|
+
type: "object",
|
|
8795
|
+
additionalProperties: false,
|
|
8796
|
+
required: ["id", "path", "mime", "checksum", "bytes"],
|
|
8797
|
+
properties: {
|
|
8798
|
+
id: { type: "string", minLength: 1 },
|
|
8799
|
+
path: { type: "string", minLength: 1 },
|
|
8800
|
+
mime: { type: ["string", "null"] },
|
|
8801
|
+
checksum: { type: "string", minLength: 1 },
|
|
8802
|
+
bytes: { type: "integer", minimum: 0 }
|
|
8803
|
+
}
|
|
8804
|
+
},
|
|
8805
|
+
attachmentProjection: {
|
|
8806
|
+
type: "object",
|
|
8807
|
+
additionalProperties: false,
|
|
8808
|
+
required: ["extracted_text", "attachment"],
|
|
8809
|
+
properties: {
|
|
8810
|
+
extracted_text: { type: ["string", "null"] },
|
|
8811
|
+
attachment: { $ref: "#/$defs/attachmentReference" }
|
|
8812
|
+
}
|
|
8813
|
+
},
|
|
8814
|
+
note: {
|
|
8815
|
+
type: "object",
|
|
8816
|
+
additionalProperties: false,
|
|
8817
|
+
description: "One row of the shard `notes` component. Field names are the server shard contract, not the PGlite table column names.",
|
|
8818
|
+
required: [
|
|
8819
|
+
"id",
|
|
8820
|
+
"title",
|
|
8821
|
+
"original_content",
|
|
8822
|
+
"revised_content",
|
|
8823
|
+
"format",
|
|
8824
|
+
"source",
|
|
8825
|
+
"starred",
|
|
8826
|
+
"archived",
|
|
8827
|
+
"tags",
|
|
8828
|
+
"created_at",
|
|
8829
|
+
"updated_at"
|
|
8830
|
+
],
|
|
8831
|
+
properties: {
|
|
8832
|
+
id: { type: "string", minLength: 1 },
|
|
8833
|
+
title: { type: ["string", "null"] },
|
|
8834
|
+
original_content: { type: "string" },
|
|
8835
|
+
revised_content: { type: ["string", "null"] },
|
|
8836
|
+
collection_id: { type: ["string", "null"] },
|
|
8837
|
+
attachments: {
|
|
8838
|
+
type: "array",
|
|
8839
|
+
items: { $ref: "#/$defs/attachmentProjection" }
|
|
8840
|
+
},
|
|
8841
|
+
format: { type: "string" },
|
|
8842
|
+
source: { type: "string" },
|
|
8843
|
+
starred: { type: "boolean" },
|
|
8844
|
+
archived: { type: "boolean" },
|
|
8845
|
+
tags: { type: "array", items: { type: "string" } },
|
|
8846
|
+
created_at: { $ref: "#/$defs/isoDateTime" },
|
|
8847
|
+
updated_at: { $ref: "#/$defs/isoDateTime" },
|
|
8848
|
+
deleted_at: { type: ["string", "null"] }
|
|
8849
|
+
}
|
|
8850
|
+
},
|
|
8851
|
+
collection: {
|
|
8852
|
+
type: "object",
|
|
8853
|
+
additionalProperties: false,
|
|
8854
|
+
required: ["id", "name", "description", "parent_id", "created_at"],
|
|
8855
|
+
properties: {
|
|
8856
|
+
id: { type: "string", minLength: 1 },
|
|
8857
|
+
name: { type: "string" },
|
|
8858
|
+
description: { type: ["string", "null"] },
|
|
8859
|
+
parent_id: { type: ["string", "null"] },
|
|
8860
|
+
created_at: { $ref: "#/$defs/isoDateTime" },
|
|
8861
|
+
note_count: { type: "integer", minimum: 0 }
|
|
8862
|
+
}
|
|
8863
|
+
},
|
|
8864
|
+
tag: {
|
|
8865
|
+
type: "object",
|
|
8866
|
+
additionalProperties: false,
|
|
8867
|
+
required: ["name", "created_at"],
|
|
8868
|
+
properties: {
|
|
8869
|
+
name: { type: "string", minLength: 1 },
|
|
8870
|
+
created_at: { $ref: "#/$defs/isoDateTime" }
|
|
8871
|
+
}
|
|
8872
|
+
},
|
|
8873
|
+
template: {
|
|
8874
|
+
type: "object",
|
|
8875
|
+
additionalProperties: false,
|
|
8876
|
+
required: ["id", "name", "description", "content", "format", "default_tags", "collection_id", "created_at", "updated_at"],
|
|
8877
|
+
properties: {
|
|
8878
|
+
id: { type: "string", minLength: 1 },
|
|
8879
|
+
name: { type: "string" },
|
|
8880
|
+
description: { type: ["string", "null"] },
|
|
8881
|
+
content: { type: "string" },
|
|
8882
|
+
format: { type: "string" },
|
|
8883
|
+
default_tags: { $ref: "#/$defs/stringArray" },
|
|
8884
|
+
collection_id: { type: ["string", "null"] },
|
|
8885
|
+
created_at: { $ref: "#/$defs/isoDateTime" },
|
|
8886
|
+
updated_at: { $ref: "#/$defs/isoDateTime" }
|
|
8887
|
+
}
|
|
8888
|
+
},
|
|
8889
|
+
link: {
|
|
8890
|
+
type: "object",
|
|
8891
|
+
additionalProperties: false,
|
|
8892
|
+
required: ["id", "from_note_id", "to_note_id", "to_url", "kind", "score", "created_at", "metadata"],
|
|
8893
|
+
properties: {
|
|
8894
|
+
id: { type: "string", minLength: 1 },
|
|
8895
|
+
from_note_id: { type: "string", minLength: 1 },
|
|
8896
|
+
to_note_id: { type: ["string", "null"] },
|
|
8897
|
+
to_url: { type: ["string", "null"] },
|
|
8898
|
+
kind: { type: "string" },
|
|
8899
|
+
score: { type: ["number", "null"] },
|
|
8900
|
+
created_at: { $ref: "#/$defs/isoDateTime" },
|
|
8901
|
+
metadata: { $ref: "#/$defs/nullableJsonObject" }
|
|
8902
|
+
}
|
|
8903
|
+
},
|
|
8904
|
+
embeddingSet: {
|
|
8905
|
+
type: "object",
|
|
8906
|
+
additionalProperties: false,
|
|
8907
|
+
required: ["id", "name", "slug", "description", "purpose", "document_count", "embedding_count", "is_system", "keywords", "model", "dimension"],
|
|
8908
|
+
properties: {
|
|
8909
|
+
id: { type: "string", minLength: 1 },
|
|
8910
|
+
name: { type: "string" },
|
|
8911
|
+
slug: { type: ["string", "null"] },
|
|
8912
|
+
description: { type: ["string", "null"] },
|
|
8913
|
+
purpose: { type: ["string", "null"] },
|
|
8914
|
+
document_count: { type: "integer", minimum: 0 },
|
|
8915
|
+
embedding_count: { type: "integer", minimum: 0 },
|
|
8916
|
+
is_system: { type: "boolean" },
|
|
8917
|
+
keywords: { $ref: "#/$defs/stringArray" },
|
|
8918
|
+
model: { type: "string" },
|
|
8919
|
+
dimension: { type: "integer", minimum: 1 },
|
|
8920
|
+
kind: { enum: ["physical", "filter", "virtual"] },
|
|
8921
|
+
mode: { type: ["string", "null"] },
|
|
8922
|
+
truncate_dimension: { type: ["integer", "null"], minimum: 1 },
|
|
8923
|
+
criteria: { $ref: "#/$defs/nullableJsonObject" },
|
|
8924
|
+
source: { $ref: "#/$defs/nullableJsonObject" },
|
|
8925
|
+
compatibility: { $ref: "#/$defs/nullableJsonObject" },
|
|
8926
|
+
materialization: { $ref: "#/$defs/nullableJsonObject" },
|
|
8927
|
+
freshness: { $ref: "#/$defs/nullableJsonObject" },
|
|
8928
|
+
created_at: { $ref: "#/$defs/isoDateTime" },
|
|
8929
|
+
updated_at: { $ref: "#/$defs/isoDateTime" }
|
|
8930
|
+
}
|
|
8931
|
+
},
|
|
8932
|
+
embeddingSetMember: {
|
|
8933
|
+
type: "object",
|
|
8934
|
+
additionalProperties: false,
|
|
8935
|
+
required: ["embedding_set_id", "note_id", "membership_type", "added_at", "added_by"],
|
|
8936
|
+
properties: {
|
|
8937
|
+
embedding_set_id: { type: "string", minLength: 1 },
|
|
8938
|
+
note_id: { type: "string", minLength: 1 },
|
|
8939
|
+
membership_type: { type: "string" },
|
|
8940
|
+
added_at: { $ref: "#/$defs/isoDateTime" },
|
|
8941
|
+
added_by: { type: ["string", "null"] }
|
|
8942
|
+
}
|
|
8943
|
+
},
|
|
8944
|
+
embeddingConfig: {
|
|
8945
|
+
type: "object",
|
|
8946
|
+
additionalProperties: false,
|
|
8947
|
+
required: ["id", "name", "description", "model", "dimension", "chunk_size", "chunk_overlap", "is_default"],
|
|
8948
|
+
properties: {
|
|
8949
|
+
id: { type: "string", minLength: 1 },
|
|
8950
|
+
name: { type: "string" },
|
|
8951
|
+
description: { type: ["string", "null"] },
|
|
8952
|
+
model: { type: "string" },
|
|
8953
|
+
dimension: { type: "integer", minimum: 1 },
|
|
8954
|
+
chunk_size: { type: "integer", minimum: 1 },
|
|
8955
|
+
chunk_overlap: { type: "integer", minimum: 0 },
|
|
8956
|
+
is_default: { type: "boolean" }
|
|
8957
|
+
}
|
|
8958
|
+
},
|
|
8959
|
+
embedding: {
|
|
8960
|
+
type: "object",
|
|
8961
|
+
additionalProperties: false,
|
|
8962
|
+
required: ["id", "note_id", "chunk_index", "text", "vector", "model"],
|
|
8963
|
+
properties: {
|
|
8964
|
+
id: { type: "string", minLength: 1 },
|
|
8965
|
+
note_id: { type: "string", minLength: 1 },
|
|
8966
|
+
chunk_index: { type: "integer", minimum: 0 },
|
|
8967
|
+
text: { type: "string" },
|
|
8968
|
+
vector: { type: "array", items: { type: "number" } },
|
|
8969
|
+
model: { type: "string" },
|
|
8970
|
+
embedding_set_id: { type: "string", minLength: 1 },
|
|
8971
|
+
created_at: { $ref: "#/$defs/isoDateTime" }
|
|
8972
|
+
}
|
|
8973
|
+
},
|
|
8974
|
+
skosScheme: {
|
|
8975
|
+
type: "object",
|
|
8976
|
+
additionalProperties: false,
|
|
8977
|
+
required: ["id", "title", "description", "created_at", "updated_at"],
|
|
8978
|
+
properties: {
|
|
8979
|
+
id: { type: "string", minLength: 1 },
|
|
8980
|
+
title: { type: "string" },
|
|
8981
|
+
description: { type: ["string", "null"] },
|
|
8982
|
+
created_at: { $ref: "#/$defs/isoDateTime" },
|
|
8983
|
+
updated_at: { $ref: "#/$defs/isoDateTime" }
|
|
8984
|
+
}
|
|
8985
|
+
},
|
|
8986
|
+
skosConcept: {
|
|
8987
|
+
type: "object",
|
|
8988
|
+
additionalProperties: false,
|
|
8989
|
+
required: ["id", "scheme_id", "pref_label", "alt_labels", "definition", "created_at", "updated_at"],
|
|
8990
|
+
properties: {
|
|
8991
|
+
id: { type: "string", minLength: 1 },
|
|
8992
|
+
scheme_id: { type: "string", minLength: 1 },
|
|
8993
|
+
pref_label: { type: "string" },
|
|
8994
|
+
alt_labels: { $ref: "#/$defs/stringArray" },
|
|
8995
|
+
definition: { type: ["string", "null"] },
|
|
8996
|
+
created_at: { $ref: "#/$defs/isoDateTime" },
|
|
8997
|
+
updated_at: { $ref: "#/$defs/isoDateTime" }
|
|
8998
|
+
}
|
|
8999
|
+
},
|
|
9000
|
+
skosRelation: {
|
|
9001
|
+
type: "object",
|
|
9002
|
+
additionalProperties: false,
|
|
9003
|
+
required: ["id", "source_concept_id", "target_concept_id", "relation_type", "created_at"],
|
|
9004
|
+
properties: {
|
|
9005
|
+
id: { type: "string", minLength: 1 },
|
|
9006
|
+
source_concept_id: { type: "string", minLength: 1 },
|
|
9007
|
+
target_concept_id: { type: "string", minLength: 1 },
|
|
9008
|
+
relation_type: { enum: ["broader", "narrower", "related"] },
|
|
9009
|
+
created_at: { $ref: "#/$defs/isoDateTime" }
|
|
9010
|
+
}
|
|
9011
|
+
},
|
|
9012
|
+
noteSkosTag: {
|
|
9013
|
+
type: "object",
|
|
9014
|
+
additionalProperties: false,
|
|
9015
|
+
required: ["id", "note_id", "concept_id", "created_at"],
|
|
9016
|
+
properties: {
|
|
9017
|
+
id: { type: "string", minLength: 1 },
|
|
9018
|
+
note_id: { type: "string", minLength: 1 },
|
|
9019
|
+
concept_id: { type: "string", minLength: 1 },
|
|
9020
|
+
created_at: { $ref: "#/$defs/isoDateTime" }
|
|
9021
|
+
}
|
|
9022
|
+
},
|
|
9023
|
+
provenanceEdge: {
|
|
9024
|
+
type: "object",
|
|
9025
|
+
additionalProperties: false,
|
|
9026
|
+
required: ["id", "entity_type", "entity_id", "activity", "agent", "started_at", "ended_at", "attributes"],
|
|
9027
|
+
properties: {
|
|
9028
|
+
id: { type: "string", minLength: 1 },
|
|
9029
|
+
entity_type: { type: "string" },
|
|
9030
|
+
entity_id: { type: "string", minLength: 1 },
|
|
9031
|
+
activity: { type: "string" },
|
|
9032
|
+
agent: { type: "string" },
|
|
9033
|
+
started_at: { $ref: "#/$defs/isoDateTime" },
|
|
9034
|
+
ended_at: { type: ["string", "null"] },
|
|
9035
|
+
attributes: { $ref: "#/$defs/nullableJsonObject" }
|
|
9036
|
+
}
|
|
9037
|
+
},
|
|
9038
|
+
artifactFreshness: {
|
|
9039
|
+
type: "object",
|
|
9040
|
+
additionalProperties: false,
|
|
9041
|
+
required: ["status"],
|
|
9042
|
+
properties: {
|
|
9043
|
+
status: { enum: ["fresh", "stale", "unknown"] },
|
|
9044
|
+
checked_at: { $ref: "#/$defs/isoDateTime" },
|
|
9045
|
+
stale_reason: { type: "string" },
|
|
9046
|
+
source_hashes: { $ref: "#/$defs/jsonObject" }
|
|
9047
|
+
}
|
|
9048
|
+
},
|
|
9049
|
+
graphSource: {
|
|
9050
|
+
type: "object",
|
|
9051
|
+
additionalProperties: false,
|
|
9052
|
+
required: ["id", "name", "kind", "input_hash", "freshness", "created_at"],
|
|
9053
|
+
properties: {
|
|
9054
|
+
id: { type: "string", minLength: 1 },
|
|
9055
|
+
name: { type: "string" },
|
|
9056
|
+
kind: { enum: ["link", "similarity", "search", "manual", "imported"] },
|
|
9057
|
+
source_table: { type: ["string", "null"] },
|
|
9058
|
+
embedding_set_id: { type: ["string", "null"] },
|
|
9059
|
+
virtual_set_id: { type: ["string", "null"] },
|
|
9060
|
+
model: { type: ["string", "null"] },
|
|
9061
|
+
dimension: { type: ["integer", "null"], minimum: 1 },
|
|
9062
|
+
truncate_dimension: { type: ["integer", "null"], minimum: 1 },
|
|
9063
|
+
metric: { type: ["string", "null"] },
|
|
9064
|
+
algorithm: { type: ["string", "null"] },
|
|
9065
|
+
parameters: { $ref: "#/$defs/jsonObject" },
|
|
9066
|
+
input_hash: { type: "string" },
|
|
9067
|
+
freshness: { $ref: "#/$defs/artifactFreshness" },
|
|
9068
|
+
created_at: { $ref: "#/$defs/isoDateTime" }
|
|
9069
|
+
}
|
|
9070
|
+
},
|
|
9071
|
+
graphEdge: {
|
|
9072
|
+
type: "object",
|
|
9073
|
+
additionalProperties: false,
|
|
9074
|
+
required: ["graph_source_id", "from_note_id", "to_note_id", "weight", "kind"],
|
|
9075
|
+
properties: {
|
|
9076
|
+
graph_source_id: { type: "string", minLength: 1 },
|
|
9077
|
+
from_note_id: { type: "string", minLength: 1 },
|
|
9078
|
+
to_note_id: { type: "string", minLength: 1 },
|
|
9079
|
+
weight: { type: "number" },
|
|
9080
|
+
kind: { enum: ["link", "similarity", "manual"] },
|
|
9081
|
+
rank: { type: ["integer", "null"], minimum: 0 },
|
|
9082
|
+
metadata: { $ref: "#/$defs/jsonObject" }
|
|
9083
|
+
}
|
|
9084
|
+
},
|
|
9085
|
+
community: {
|
|
9086
|
+
type: "object",
|
|
9087
|
+
additionalProperties: false,
|
|
9088
|
+
required: ["id"],
|
|
9089
|
+
properties: {
|
|
9090
|
+
id: { type: "string", minLength: 1 },
|
|
9091
|
+
label: { type: ["string", "null"] },
|
|
9092
|
+
rank: { type: ["integer", "null"], minimum: 0 },
|
|
9093
|
+
size: { type: ["integer", "null"], minimum: 0 },
|
|
9094
|
+
confidence: { type: ["number", "null"] },
|
|
9095
|
+
representative_note_ids: { $ref: "#/$defs/stringArray" },
|
|
9096
|
+
metadata: { $ref: "#/$defs/jsonObject" }
|
|
9097
|
+
}
|
|
9098
|
+
},
|
|
9099
|
+
communitySet: {
|
|
9100
|
+
type: "object",
|
|
9101
|
+
additionalProperties: false,
|
|
9102
|
+
required: ["id", "graph_source_id", "name", "source_type", "input_hash", "freshness", "communities", "created_at"],
|
|
9103
|
+
properties: {
|
|
9104
|
+
id: { type: "string", minLength: 1 },
|
|
9105
|
+
graph_source_id: { type: "string", minLength: 1 },
|
|
9106
|
+
name: { type: "string" },
|
|
9107
|
+
source_type: { enum: ["precomputed", "dynamic-snapshot", "user-authored", "imported"] },
|
|
9108
|
+
algorithm: { type: ["string", "null"] },
|
|
9109
|
+
parameters: { $ref: "#/$defs/jsonObject" },
|
|
9110
|
+
input_hash: { type: "string" },
|
|
9111
|
+
freshness: { $ref: "#/$defs/artifactFreshness" },
|
|
9112
|
+
communities: {
|
|
9113
|
+
type: "array",
|
|
9114
|
+
items: { $ref: "#/$defs/community" }
|
|
9115
|
+
},
|
|
9116
|
+
created_at: { $ref: "#/$defs/isoDateTime" }
|
|
9117
|
+
}
|
|
9118
|
+
},
|
|
9119
|
+
communityAssignment: {
|
|
9120
|
+
type: "object",
|
|
9121
|
+
additionalProperties: false,
|
|
9122
|
+
required: ["community_set_id", "community_id", "note_id", "source_type"],
|
|
9123
|
+
properties: {
|
|
9124
|
+
community_set_id: { type: "string", minLength: 1 },
|
|
9125
|
+
community_id: { type: "string", minLength: 1 },
|
|
9126
|
+
note_id: { type: "string", minLength: 1 },
|
|
9127
|
+
confidence: { type: ["number", "null"] },
|
|
9128
|
+
source_type: { enum: ["precomputed", "dynamic-snapshot", "user-authored", "imported"] },
|
|
9129
|
+
metadata: { $ref: "#/$defs/jsonObject" }
|
|
9130
|
+
}
|
|
9131
|
+
}
|
|
9132
|
+
}
|
|
9133
|
+
};
|
|
9134
|
+
|
|
9135
|
+
// src/shard/schema-validator.ts
|
|
9136
|
+
var decoder3 = new TextDecoder();
|
|
9137
|
+
var COMPONENT_SCHEMA_DEFS = {
|
|
9138
|
+
notes: "note",
|
|
9139
|
+
collections: "collection",
|
|
9140
|
+
tags: "tag",
|
|
9141
|
+
templates: "template",
|
|
9142
|
+
links: "link",
|
|
9143
|
+
embedding_sets: "embeddingSet",
|
|
9144
|
+
embedding_set_members: "embeddingSetMember",
|
|
9145
|
+
embedding_configs: "embeddingConfig",
|
|
9146
|
+
embeddings: "embedding",
|
|
9147
|
+
skos_schemes: "skosScheme",
|
|
9148
|
+
skos_concepts: "skosConcept",
|
|
9149
|
+
skos_relations: "skosRelation",
|
|
9150
|
+
note_skos_tags: "noteSkosTag",
|
|
9151
|
+
provenance_edges: "provenanceEdge",
|
|
9152
|
+
graph_sources: "graphSource",
|
|
9153
|
+
graph_edges: "graphEdge",
|
|
9154
|
+
communities: "communitySet",
|
|
9155
|
+
community_assignments: "communityAssignment"
|
|
9156
|
+
};
|
|
9157
|
+
var COMPONENT_FILES = {
|
|
9158
|
+
notes: { file: "notes.jsonl", encoding: "jsonl" },
|
|
9159
|
+
collections: { file: "collections.json", encoding: "json-array" },
|
|
9160
|
+
tags: { file: "tags.json", encoding: "json-array" },
|
|
9161
|
+
templates: { file: "templates.json", encoding: "json-array" },
|
|
9162
|
+
links: { file: "links.jsonl", encoding: "jsonl" },
|
|
9163
|
+
embedding_sets: { file: "embedding_sets.json", encoding: "json-array" },
|
|
9164
|
+
embedding_set_members: { file: "embedding_set_members.jsonl", encoding: "jsonl" },
|
|
9165
|
+
embedding_configs: { file: "embedding_configs.json", encoding: "json-array" },
|
|
9166
|
+
embeddings: { file: "embeddings.jsonl", encoding: "jsonl" },
|
|
9167
|
+
skos_schemes: { file: "skos_schemes.json", encoding: "json-array" },
|
|
9168
|
+
skos_concepts: { file: "skos_concepts.json", encoding: "json-array" },
|
|
9169
|
+
skos_relations: { file: "skos_relations.jsonl", encoding: "jsonl" },
|
|
9170
|
+
note_skos_tags: { file: "note_skos_tags.jsonl", encoding: "jsonl" },
|
|
9171
|
+
provenance_edges: { file: "provenance_edges.jsonl", encoding: "jsonl" },
|
|
9172
|
+
graph_sources: { file: "graph_sources.json", encoding: "json-array" },
|
|
9173
|
+
graph_edges: { file: "graph_edges.jsonl", encoding: "jsonl" },
|
|
9174
|
+
communities: { file: "communities.json", encoding: "json-array" },
|
|
9175
|
+
community_assignments: { file: "community_assignments.jsonl", encoding: "jsonl" }
|
|
9176
|
+
};
|
|
9177
|
+
var ajvInstance;
|
|
9178
|
+
var validators = /* @__PURE__ */ new Map();
|
|
9179
|
+
function getKnowledgeShardSchema() {
|
|
9180
|
+
return knowledge_shard_schema_default;
|
|
9181
|
+
}
|
|
9182
|
+
function getAjv() {
|
|
9183
|
+
if (!ajvInstance) {
|
|
9184
|
+
ajvInstance = new Ajv2020({
|
|
9185
|
+
allErrors: true,
|
|
9186
|
+
strict: true,
|
|
9187
|
+
validateFormats: false
|
|
9188
|
+
});
|
|
9189
|
+
ajvInstance.addSchema(knowledge_shard_schema_default);
|
|
9190
|
+
}
|
|
9191
|
+
return ajvInstance;
|
|
9192
|
+
}
|
|
9193
|
+
function validatorFor(defName) {
|
|
9194
|
+
const cached = validators.get(defName);
|
|
9195
|
+
if (cached) return cached;
|
|
9196
|
+
const validator = getAjv().getSchema(`${knowledge_shard_schema_default.$id}#/$defs/${defName}`) ?? getAjv().compile({ $ref: `${knowledge_shard_schema_default.$id}#/$defs/${defName}` });
|
|
9197
|
+
validators.set(defName, validator);
|
|
9198
|
+
return validator;
|
|
9199
|
+
}
|
|
9200
|
+
function formatErrors(errors) {
|
|
9201
|
+
return (errors ?? []).map((error) => {
|
|
9202
|
+
const path = error.instancePath || "(root)";
|
|
9203
|
+
return `${path} ${error.message ?? "is invalid"}`;
|
|
9204
|
+
});
|
|
9205
|
+
}
|
|
9206
|
+
function validateShardManifest(value) {
|
|
9207
|
+
const validate = validatorFor("manifest");
|
|
9208
|
+
const valid = validate(value);
|
|
9209
|
+
return { valid, errors: formatErrors(validate.errors) };
|
|
9210
|
+
}
|
|
9211
|
+
function parseJsonArray(bytes, path) {
|
|
9212
|
+
if (!bytes) return { records: [], errors: [] };
|
|
9213
|
+
try {
|
|
9214
|
+
const value = JSON.parse(decoder3.decode(bytes));
|
|
9215
|
+
if (!Array.isArray(value)) return { records: [], errors: [`${path} must be a JSON array`] };
|
|
9216
|
+
return { records: value, errors: [] };
|
|
9217
|
+
} catch (error) {
|
|
9218
|
+
return { records: [], errors: [`${path} failed to parse: ${error instanceof Error ? error.message : String(error)}`] };
|
|
9219
|
+
}
|
|
9220
|
+
}
|
|
9221
|
+
function parseJsonl(bytes, path) {
|
|
9222
|
+
if (!bytes) return { records: [], errors: [] };
|
|
9223
|
+
const text = decoder3.decode(bytes).trim();
|
|
9224
|
+
if (!text) return { records: [], errors: [] };
|
|
9225
|
+
const records = [];
|
|
9226
|
+
const errors = [];
|
|
9227
|
+
for (const [index, line] of text.split("\n").entries()) {
|
|
9228
|
+
try {
|
|
9229
|
+
records.push(JSON.parse(line));
|
|
9230
|
+
} catch (error) {
|
|
9231
|
+
errors.push(`${path}:${index + 1} failed to parse: ${error instanceof Error ? error.message : String(error)}`);
|
|
9232
|
+
}
|
|
9233
|
+
}
|
|
9234
|
+
return { records, errors };
|
|
9235
|
+
}
|
|
9236
|
+
function unpackShardFiles(input) {
|
|
9237
|
+
if (input instanceof Map) return input;
|
|
9238
|
+
return unpackTarGz(input instanceof ArrayBuffer ? new Uint8Array(input) : input);
|
|
9239
|
+
}
|
|
9240
|
+
function validateComponentRecords(component, records, path) {
|
|
9241
|
+
const errors = [];
|
|
9242
|
+
for (const [index, record] of records.entries()) {
|
|
9243
|
+
const result = validateShardComponentRecord(component, record);
|
|
9244
|
+
if (!result.valid) {
|
|
9245
|
+
errors.push(...result.errors.map((error) => `${path}[${index}] ${error}`));
|
|
9246
|
+
}
|
|
9247
|
+
}
|
|
9248
|
+
return errors;
|
|
9249
|
+
}
|
|
9250
|
+
function validateShardArchive(input) {
|
|
9251
|
+
let files;
|
|
9252
|
+
try {
|
|
9253
|
+
files = unpackShardFiles(input);
|
|
9254
|
+
} catch (error) {
|
|
9255
|
+
return {
|
|
9256
|
+
valid: false,
|
|
9257
|
+
errors: [`archive failed to unpack: ${error instanceof Error ? error.message : String(error)}`]
|
|
9258
|
+
};
|
|
9259
|
+
}
|
|
9260
|
+
const manifestBytes = files.get("manifest.json");
|
|
9261
|
+
if (!manifestBytes) return { valid: false, errors: ["manifest.json is missing"] };
|
|
9262
|
+
let manifest;
|
|
9263
|
+
try {
|
|
9264
|
+
manifest = JSON.parse(decoder3.decode(manifestBytes));
|
|
9265
|
+
} catch (error) {
|
|
9266
|
+
return {
|
|
9267
|
+
valid: false,
|
|
9268
|
+
errors: [`manifest.json failed to parse: ${error instanceof Error ? error.message : String(error)}`]
|
|
9269
|
+
};
|
|
9270
|
+
}
|
|
9271
|
+
const errors = [];
|
|
9272
|
+
const manifestResult = validateShardManifest(manifest);
|
|
9273
|
+
if (!manifestResult.valid) {
|
|
9274
|
+
errors.push(...manifestResult.errors.map((error) => `manifest.json ${error}`));
|
|
9275
|
+
}
|
|
9276
|
+
const componentsToValidate = new Set(manifest.components);
|
|
9277
|
+
for (const [component, spec] of Object.entries(COMPONENT_FILES)) {
|
|
9278
|
+
if (files.has(spec.file)) componentsToValidate.add(component);
|
|
9279
|
+
}
|
|
9280
|
+
for (const component of componentsToValidate) {
|
|
9281
|
+
const spec = COMPONENT_FILES[component];
|
|
9282
|
+
if (!spec) continue;
|
|
9283
|
+
if (component === "notes" && manifest.layout?.clusters?.notes?.length) {
|
|
9284
|
+
for (const cluster of manifest.layout.clusters.notes) {
|
|
9285
|
+
const parsed2 = parseJsonl(files.get(cluster.href), cluster.href);
|
|
9286
|
+
errors.push(...parsed2.errors);
|
|
9287
|
+
errors.push(...validateComponentRecords("notes", parsed2.records, cluster.href));
|
|
9288
|
+
}
|
|
9289
|
+
continue;
|
|
9290
|
+
}
|
|
9291
|
+
const parsed = spec.encoding === "json-array" ? parseJsonArray(files.get(spec.file), spec.file) : parseJsonl(files.get(spec.file), spec.file);
|
|
9292
|
+
errors.push(...parsed.errors);
|
|
9293
|
+
errors.push(...validateComponentRecords(component, parsed.records, spec.file));
|
|
9294
|
+
}
|
|
9295
|
+
return { valid: errors.length === 0, errors };
|
|
9296
|
+
}
|
|
9297
|
+
function validateShardComponentRecord(component, value) {
|
|
9298
|
+
const defName = COMPONENT_SCHEMA_DEFS[component];
|
|
9299
|
+
const validate = validatorFor(defName);
|
|
9300
|
+
const valid = validate(value);
|
|
9301
|
+
return { valid, errors: formatErrors(validate.errors) };
|
|
9302
|
+
}
|
|
9303
|
+
function assertShardComponentRecord(component, value) {
|
|
9304
|
+
const result = validateShardComponentRecord(component, value);
|
|
9305
|
+
if (!result.valid) {
|
|
9306
|
+
throw new Error(`Invalid shard ${component} record:
|
|
9307
|
+
${result.errors.join("\n")}`);
|
|
9308
|
+
}
|
|
7715
9309
|
}
|
|
7716
9310
|
|
|
7717
9311
|
// src/shard/shard-reader.ts
|
|
7718
|
-
var
|
|
9312
|
+
var decoder4 = new TextDecoder();
|
|
7719
9313
|
function assertSafeComponentName(filename) {
|
|
7720
9314
|
if (filename.length === 0 || filename.startsWith("/") || filename.includes("\\") || filename.includes("\0") || filename.includes(":") || filename.split("/").some((segment) => segment === "..")) {
|
|
7721
9315
|
throw new Error(`Refusing to read unsafe shard component path: ${JSON.stringify(filename)}`);
|
|
7722
9316
|
}
|
|
7723
9317
|
}
|
|
7724
|
-
function parseJsonlBytes(data) {
|
|
7725
|
-
if (!data || data.byteLength === 0) return [];
|
|
7726
|
-
return decoder2.decode(data).split("\n").filter((line) => line.trim().length > 0).map((line) => JSON.parse(line));
|
|
7727
|
-
}
|
|
7728
|
-
function parseJsonArrayBytes(data) {
|
|
7729
|
-
if (!data || data.byteLength === 0) return [];
|
|
7730
|
-
return JSON.parse(decoder2.decode(data));
|
|
7731
|
-
}
|
|
7732
9318
|
var DEFAULT_WEIGHTS = { title: 4, content: 1, tag: 2 };
|
|
7733
9319
|
async function toBytes(blobOrBytes) {
|
|
7734
9320
|
if (blobOrBytes instanceof Uint8Array) return blobOrBytes;
|
|
@@ -7741,8 +9327,13 @@ var PackedComponentStore = class {
|
|
|
7741
9327
|
this.files = files;
|
|
7742
9328
|
this.manifest = manifest;
|
|
7743
9329
|
}
|
|
7744
|
-
read(filename) {
|
|
7745
|
-
|
|
9330
|
+
async read(filename) {
|
|
9331
|
+
const bytes = this.files.get(filename);
|
|
9332
|
+
const expectedChecksum = this.manifest.checksums?.[filename];
|
|
9333
|
+
if (bytes && expectedChecksum && await sha256Hex(bytes) !== expectedChecksum) {
|
|
9334
|
+
throw new Error(`Checksum validation failed for shard component: ${filename}`);
|
|
9335
|
+
}
|
|
9336
|
+
return bytes;
|
|
7746
9337
|
}
|
|
7747
9338
|
};
|
|
7748
9339
|
var UrlComponentStore = class {
|
|
@@ -7750,10 +9341,12 @@ var UrlComponentStore = class {
|
|
|
7750
9341
|
baseUrl;
|
|
7751
9342
|
fetchImpl;
|
|
7752
9343
|
cache = /* @__PURE__ */ new Map();
|
|
7753
|
-
|
|
9344
|
+
maxComponentBytes;
|
|
9345
|
+
constructor(baseUrl, fetchImpl, manifest, maxComponentBytes) {
|
|
7754
9346
|
this.baseUrl = baseUrl.replace(/\/$/, "");
|
|
7755
9347
|
this.fetchImpl = fetchImpl;
|
|
7756
9348
|
this.manifest = manifest;
|
|
9349
|
+
this.maxComponentBytes = maxComponentBytes;
|
|
7757
9350
|
}
|
|
7758
9351
|
async read(filename) {
|
|
7759
9352
|
assertSafeComponentName(filename);
|
|
@@ -7763,12 +9356,47 @@ var UrlComponentStore = class {
|
|
|
7763
9356
|
this.cache.set(filename, void 0);
|
|
7764
9357
|
return void 0;
|
|
7765
9358
|
}
|
|
7766
|
-
const
|
|
9359
|
+
const declaredLength = Number(response.headers.get("content-length"));
|
|
9360
|
+
if (Number.isFinite(declaredLength) && declaredLength > this.maxComponentBytes) {
|
|
9361
|
+
throw new Error(`Shard component ${filename} exceeds cap ${this.maxComponentBytes} bytes`);
|
|
9362
|
+
}
|
|
9363
|
+
const chunks = [];
|
|
9364
|
+
let total = 0;
|
|
9365
|
+
if (response.body) {
|
|
9366
|
+
const reader = response.body.getReader();
|
|
9367
|
+
while (true) {
|
|
9368
|
+
const { done, value } = await reader.read();
|
|
9369
|
+
if (done) break;
|
|
9370
|
+
total += value.byteLength;
|
|
9371
|
+
if (total > this.maxComponentBytes) {
|
|
9372
|
+
await reader.cancel();
|
|
9373
|
+
throw new Error(`Shard component ${filename} exceeds cap ${this.maxComponentBytes} bytes`);
|
|
9374
|
+
}
|
|
9375
|
+
chunks.push(value);
|
|
9376
|
+
}
|
|
9377
|
+
} else {
|
|
9378
|
+
const value = new Uint8Array(await response.arrayBuffer());
|
|
9379
|
+
total = value.byteLength;
|
|
9380
|
+
if (total > this.maxComponentBytes) {
|
|
9381
|
+
throw new Error(`Shard component ${filename} exceeds cap ${this.maxComponentBytes} bytes`);
|
|
9382
|
+
}
|
|
9383
|
+
chunks.push(value);
|
|
9384
|
+
}
|
|
9385
|
+
const bytes = new Uint8Array(total);
|
|
9386
|
+
let offset = 0;
|
|
9387
|
+
for (const chunk of chunks) {
|
|
9388
|
+
bytes.set(chunk, offset);
|
|
9389
|
+
offset += chunk.byteLength;
|
|
9390
|
+
}
|
|
9391
|
+
const expectedChecksum = this.manifest.checksums?.[filename];
|
|
9392
|
+
if (expectedChecksum && await sha256Hex(bytes) !== expectedChecksum) {
|
|
9393
|
+
throw new Error(`Checksum validation failed for shard component: ${filename}`);
|
|
9394
|
+
}
|
|
7767
9395
|
this.cache.set(filename, bytes);
|
|
7768
9396
|
return bytes;
|
|
7769
9397
|
}
|
|
7770
9398
|
};
|
|
7771
|
-
async function resolveStore(source) {
|
|
9399
|
+
async function resolveStore(source, maxComponentBytes) {
|
|
7772
9400
|
if (typeof source === "object" && "baseUrl" in source) {
|
|
7773
9401
|
const fetchImpl = source.fetchImpl ?? globalThis.fetch;
|
|
7774
9402
|
const base = source.baseUrl.replace(/\/$/, "");
|
|
@@ -7777,20 +9405,20 @@ async function resolveStore(source) {
|
|
|
7777
9405
|
throw new Error(`Failed to fetch shard manifest (${manifestResponse.status}): ${base}/manifest.json`);
|
|
7778
9406
|
}
|
|
7779
9407
|
const manifest2 = await manifestResponse.json();
|
|
7780
|
-
return new UrlComponentStore(base, fetchImpl, manifest2);
|
|
9408
|
+
return new UrlComponentStore(base, fetchImpl, manifest2, maxComponentBytes);
|
|
7781
9409
|
}
|
|
7782
9410
|
const bytes = await toBytes(source);
|
|
7783
9411
|
const files = unpackTarGz(bytes);
|
|
7784
9412
|
const manifestBytes = files.get("manifest.json");
|
|
7785
9413
|
if (!manifestBytes) throw new Error("Missing manifest.json in shard archive");
|
|
7786
|
-
const manifest = JSON.parse(
|
|
9414
|
+
const manifest = JSON.parse(decoder4.decode(manifestBytes));
|
|
7787
9415
|
return new PackedComponentStore(files, manifest);
|
|
7788
9416
|
}
|
|
7789
9417
|
function tokenize(query) {
|
|
7790
9418
|
return query.toLowerCase().split(/[^a-z0-9]+/i).filter((token) => token.length > 0);
|
|
7791
9419
|
}
|
|
7792
9420
|
function noteSearchText(note) {
|
|
7793
|
-
const extractedText = note.binary_sources?.map((source) => source.extracted_text).filter(Boolean).join(" ") ?? "";
|
|
9421
|
+
const extractedText = (note.attachments ?? note.binary_sources)?.map((source) => source.extracted_text).filter(Boolean).join(" ") ?? "";
|
|
7794
9422
|
return `${note.title ?? ""} ${note.original_content} ${note.revised_content ?? ""} ${extractedText}`.toLowerCase();
|
|
7795
9423
|
}
|
|
7796
9424
|
function countOccurrences(haystack, needle) {
|
|
@@ -7812,7 +9440,7 @@ function noteMatchesTokens(note, tokens) {
|
|
|
7812
9440
|
function rankNote(note, tokens, weights) {
|
|
7813
9441
|
if (tokens.length === 0) return 0;
|
|
7814
9442
|
const title = (note.title ?? "").toLowerCase();
|
|
7815
|
-
const extractedText = note.binary_sources?.map((source) => source.extracted_text).filter(Boolean).join(" ") ?? "";
|
|
9443
|
+
const extractedText = (note.attachments ?? note.binary_sources)?.map((source) => source.extracted_text).filter(Boolean).join(" ") ?? "";
|
|
7816
9444
|
const content = `${note.original_content} ${note.revised_content ?? ""} ${extractedText}`.toLowerCase();
|
|
7817
9445
|
const tagText = note.tags.join(" ").toLowerCase();
|
|
7818
9446
|
let score = 0;
|
|
@@ -7824,7 +9452,7 @@ function rankNote(note, tokens, weights) {
|
|
|
7824
9452
|
return score;
|
|
7825
9453
|
}
|
|
7826
9454
|
function makeSnippet(note, tokens, length) {
|
|
7827
|
-
const extractedText = note.binary_sources?.map((source) => source.extracted_text).filter(Boolean).join(" ") ?? "";
|
|
9455
|
+
const extractedText = (note.attachments ?? note.binary_sources)?.map((source) => source.extracted_text).filter(Boolean).join(" ") ?? "";
|
|
7828
9456
|
const content = `${note.original_content} ${note.revised_content ?? ""} ${extractedText}`.trim();
|
|
7829
9457
|
if (tokens.length === 0) return content.slice(0, length);
|
|
7830
9458
|
const lower = content.toLowerCase();
|
|
@@ -8058,8 +9686,8 @@ var ShardReaderImpl = class {
|
|
|
8058
9686
|
}
|
|
8059
9687
|
};
|
|
8060
9688
|
async function openShard(source, options = {}) {
|
|
8061
|
-
const store = await resolveStore(source);
|
|
8062
|
-
if (store.manifest.min_reader_version && store.manifest.min_reader_version >
|
|
9689
|
+
const store = await resolveStore(source, options.maxComponentBytes ?? DEFAULT_MAX_DECOMPRESSED_BYTES);
|
|
9690
|
+
if (store.manifest.min_reader_version && compareShardVersions(store.manifest.min_reader_version, CURRENT_SHARD_VERSION) > 0) {
|
|
8063
9691
|
throw new Error(
|
|
8064
9692
|
`Shard requires reader version ${store.manifest.min_reader_version}, but this build supports ${CURRENT_SHARD_VERSION}. Import the shard instead.`
|
|
8065
9693
|
);
|
|
@@ -8068,7 +9696,7 @@ async function openShard(source, options = {}) {
|
|
|
8068
9696
|
}
|
|
8069
9697
|
|
|
8070
9698
|
// src/shard/semantic-providers.ts
|
|
8071
|
-
var
|
|
9699
|
+
var decoder5 = new TextDecoder();
|
|
8072
9700
|
function cosine(a, b) {
|
|
8073
9701
|
let dot = 0;
|
|
8074
9702
|
let normA = 0;
|
|
@@ -8092,7 +9720,7 @@ function createCosineSemanticProvider(options) {
|
|
|
8092
9720
|
vectors = [];
|
|
8093
9721
|
return;
|
|
8094
9722
|
}
|
|
8095
|
-
vectors =
|
|
9723
|
+
vectors = decoder5.decode(bytes).split("\n").filter((line) => line.trim().length > 0).map((line) => JSON.parse(line));
|
|
8096
9724
|
},
|
|
8097
9725
|
async search(query, k) {
|
|
8098
9726
|
const queryVector = await options.embedQuery(query);
|
|
@@ -8216,6 +9844,723 @@ function clearPrefetchedShard(url) {
|
|
|
8216
9844
|
warmStore.delete(url);
|
|
8217
9845
|
}
|
|
8218
9846
|
|
|
9847
|
+
// schemas/aiwg-fortemi-index-export.schema.json
|
|
9848
|
+
var aiwg_fortemi_index_export_schema_default = {
|
|
9849
|
+
$schema: "https://json-schema.org/draft/2020-12/schema",
|
|
9850
|
+
$id: "https://aiwg.io/schemas/aiwg-fortemi-index-export.json",
|
|
9851
|
+
title: "AIWG Fortemi Index Export",
|
|
9852
|
+
type: "object",
|
|
9853
|
+
additionalProperties: false,
|
|
9854
|
+
required: ["schema_version", "generated_at", "source", "items"],
|
|
9855
|
+
properties: {
|
|
9856
|
+
schema_version: {
|
|
9857
|
+
enum: ["aiwg.fortemi.index.export.v1", "aiwg.fortemi.index.export.v2"]
|
|
9858
|
+
},
|
|
9859
|
+
generated_at: {
|
|
9860
|
+
type: "string",
|
|
9861
|
+
format: "date-time"
|
|
9862
|
+
},
|
|
9863
|
+
source: {
|
|
9864
|
+
type: "object",
|
|
9865
|
+
additionalProperties: false,
|
|
9866
|
+
required: ["repo", "privacy"],
|
|
9867
|
+
properties: {
|
|
9868
|
+
repo: {
|
|
9869
|
+
type: "string",
|
|
9870
|
+
minLength: 1
|
|
9871
|
+
},
|
|
9872
|
+
privacy: {
|
|
9873
|
+
$ref: "#/$defs/privacy"
|
|
9874
|
+
},
|
|
9875
|
+
graph: {
|
|
9876
|
+
type: "string",
|
|
9877
|
+
minLength: 1
|
|
9878
|
+
}
|
|
9879
|
+
}
|
|
9880
|
+
},
|
|
9881
|
+
compatibility: {
|
|
9882
|
+
type: "object",
|
|
9883
|
+
additionalProperties: false,
|
|
9884
|
+
required: ["previous_schema_version", "strategy"],
|
|
9885
|
+
properties: {
|
|
9886
|
+
previous_schema_version: {
|
|
9887
|
+
const: "aiwg.fortemi.index.export.v1"
|
|
9888
|
+
},
|
|
9889
|
+
strategy: {
|
|
9890
|
+
const: "supported"
|
|
9891
|
+
}
|
|
9892
|
+
}
|
|
9893
|
+
},
|
|
9894
|
+
items: {
|
|
9895
|
+
type: "array",
|
|
9896
|
+
items: {
|
|
9897
|
+
$ref: "#/$defs/record"
|
|
9898
|
+
}
|
|
9899
|
+
}
|
|
9900
|
+
},
|
|
9901
|
+
allOf: [
|
|
9902
|
+
{
|
|
9903
|
+
if: {
|
|
9904
|
+
properties: {
|
|
9905
|
+
schema_version: {
|
|
9906
|
+
const: "aiwg.fortemi.index.export.v1"
|
|
9907
|
+
}
|
|
9908
|
+
}
|
|
9909
|
+
},
|
|
9910
|
+
then: {
|
|
9911
|
+
not: {
|
|
9912
|
+
required: ["compatibility"]
|
|
9913
|
+
},
|
|
9914
|
+
properties: {
|
|
9915
|
+
source: {
|
|
9916
|
+
not: {
|
|
9917
|
+
required: ["graph"]
|
|
9918
|
+
}
|
|
9919
|
+
},
|
|
9920
|
+
items: {
|
|
9921
|
+
items: {
|
|
9922
|
+
allOf: [
|
|
9923
|
+
{
|
|
9924
|
+
properties: {
|
|
9925
|
+
schema_version: {
|
|
9926
|
+
const: "aiwg.fortemi.index.record.v1"
|
|
9927
|
+
}
|
|
9928
|
+
}
|
|
9929
|
+
},
|
|
9930
|
+
{
|
|
9931
|
+
$ref: "#/$defs/v1RecordCompatibility"
|
|
9932
|
+
}
|
|
9933
|
+
]
|
|
9934
|
+
}
|
|
9935
|
+
}
|
|
9936
|
+
}
|
|
9937
|
+
}
|
|
9938
|
+
},
|
|
9939
|
+
{
|
|
9940
|
+
if: {
|
|
9941
|
+
properties: {
|
|
9942
|
+
schema_version: {
|
|
9943
|
+
const: "aiwg.fortemi.index.export.v2"
|
|
9944
|
+
}
|
|
9945
|
+
}
|
|
9946
|
+
},
|
|
9947
|
+
then: {
|
|
9948
|
+
required: ["compatibility"],
|
|
9949
|
+
properties: {
|
|
9950
|
+
source: {
|
|
9951
|
+
required: ["graph"]
|
|
9952
|
+
},
|
|
9953
|
+
items: {
|
|
9954
|
+
items: {
|
|
9955
|
+
properties: {
|
|
9956
|
+
schema_version: {
|
|
9957
|
+
const: "aiwg.fortemi.index.record.v2"
|
|
9958
|
+
}
|
|
9959
|
+
}
|
|
9960
|
+
}
|
|
9961
|
+
}
|
|
9962
|
+
}
|
|
9963
|
+
}
|
|
9964
|
+
}
|
|
9965
|
+
],
|
|
9966
|
+
$defs: {
|
|
9967
|
+
privacy: {
|
|
9968
|
+
enum: ["private", "sanitized", "public"]
|
|
9969
|
+
},
|
|
9970
|
+
recordType: {
|
|
9971
|
+
type: "string",
|
|
9972
|
+
minLength: 1
|
|
9973
|
+
},
|
|
9974
|
+
stringArray: {
|
|
9975
|
+
type: "array",
|
|
9976
|
+
items: {
|
|
9977
|
+
type: "string"
|
|
9978
|
+
}
|
|
9979
|
+
},
|
|
9980
|
+
numberArray: {
|
|
9981
|
+
type: "array",
|
|
9982
|
+
items: {
|
|
9983
|
+
type: "number"
|
|
9984
|
+
}
|
|
9985
|
+
},
|
|
9986
|
+
v1RecordCompatibility: {
|
|
9987
|
+
type: "object",
|
|
9988
|
+
not: {
|
|
9989
|
+
anyOf: [
|
|
9990
|
+
{
|
|
9991
|
+
required: ["name"]
|
|
9992
|
+
},
|
|
9993
|
+
{
|
|
9994
|
+
required: ["summary"]
|
|
9995
|
+
},
|
|
9996
|
+
{
|
|
9997
|
+
required: ["search"]
|
|
9998
|
+
},
|
|
9999
|
+
{
|
|
10000
|
+
required: ["chunks"]
|
|
10001
|
+
},
|
|
10002
|
+
{
|
|
10003
|
+
required: ["embeddings"]
|
|
10004
|
+
},
|
|
10005
|
+
{
|
|
10006
|
+
required: ["compatibility"]
|
|
10007
|
+
},
|
|
10008
|
+
{
|
|
10009
|
+
required: ["skos_concepts"]
|
|
10010
|
+
},
|
|
10011
|
+
{
|
|
10012
|
+
required: ["skos_relations"]
|
|
10013
|
+
},
|
|
10014
|
+
{
|
|
10015
|
+
required: ["provenance_events"]
|
|
10016
|
+
}
|
|
10017
|
+
]
|
|
10018
|
+
},
|
|
10019
|
+
properties: {
|
|
10020
|
+
source: {
|
|
10021
|
+
not: {
|
|
10022
|
+
anyOf: [
|
|
10023
|
+
{
|
|
10024
|
+
required: ["origin"]
|
|
10025
|
+
},
|
|
10026
|
+
{
|
|
10027
|
+
required: ["generated"]
|
|
10028
|
+
},
|
|
10029
|
+
{
|
|
10030
|
+
required: ["checksum"]
|
|
10031
|
+
},
|
|
10032
|
+
{
|
|
10033
|
+
required: ["updated_at"]
|
|
10034
|
+
}
|
|
10035
|
+
]
|
|
10036
|
+
}
|
|
10037
|
+
},
|
|
10038
|
+
relationships: {
|
|
10039
|
+
items: {
|
|
10040
|
+
not: {
|
|
10041
|
+
anyOf: [
|
|
10042
|
+
{
|
|
10043
|
+
required: ["target_path"]
|
|
10044
|
+
},
|
|
10045
|
+
{
|
|
10046
|
+
required: ["direction"]
|
|
10047
|
+
},
|
|
10048
|
+
{
|
|
10049
|
+
required: ["label"]
|
|
10050
|
+
},
|
|
10051
|
+
{
|
|
10052
|
+
required: ["confidence"]
|
|
10053
|
+
},
|
|
10054
|
+
{
|
|
10055
|
+
required: ["privacy"]
|
|
10056
|
+
},
|
|
10057
|
+
{
|
|
10058
|
+
required: ["metadata"]
|
|
10059
|
+
}
|
|
10060
|
+
]
|
|
10061
|
+
}
|
|
10062
|
+
}
|
|
10063
|
+
},
|
|
10064
|
+
privacy: {
|
|
10065
|
+
not: {
|
|
10066
|
+
required: ["locality"]
|
|
10067
|
+
}
|
|
10068
|
+
}
|
|
10069
|
+
}
|
|
10070
|
+
},
|
|
10071
|
+
record: {
|
|
10072
|
+
type: "object",
|
|
10073
|
+
additionalProperties: false,
|
|
10074
|
+
required: [
|
|
10075
|
+
"schema_version",
|
|
10076
|
+
"id",
|
|
10077
|
+
"type",
|
|
10078
|
+
"source",
|
|
10079
|
+
"title",
|
|
10080
|
+
"text",
|
|
10081
|
+
"facets",
|
|
10082
|
+
"tags",
|
|
10083
|
+
"concepts",
|
|
10084
|
+
"relationships",
|
|
10085
|
+
"provenance",
|
|
10086
|
+
"privacy",
|
|
10087
|
+
"updated_at"
|
|
10088
|
+
],
|
|
10089
|
+
properties: {
|
|
10090
|
+
schema_version: {
|
|
10091
|
+
enum: [
|
|
10092
|
+
"aiwg.fortemi.index.record.v1",
|
|
10093
|
+
"aiwg.fortemi.index.record.v2"
|
|
10094
|
+
]
|
|
10095
|
+
},
|
|
10096
|
+
id: {
|
|
10097
|
+
type: "string",
|
|
10098
|
+
minLength: 1
|
|
10099
|
+
},
|
|
10100
|
+
type: {
|
|
10101
|
+
$ref: "#/$defs/recordType"
|
|
10102
|
+
},
|
|
10103
|
+
source: {
|
|
10104
|
+
type: "object",
|
|
10105
|
+
additionalProperties: false,
|
|
10106
|
+
required: ["path", "repo_relative_path", "locator"],
|
|
10107
|
+
properties: {
|
|
10108
|
+
path: {
|
|
10109
|
+
type: "string",
|
|
10110
|
+
minLength: 1
|
|
10111
|
+
},
|
|
10112
|
+
repo_relative_path: {
|
|
10113
|
+
type: "string",
|
|
10114
|
+
minLength: 1
|
|
10115
|
+
},
|
|
10116
|
+
locator: {
|
|
10117
|
+
type: "string",
|
|
10118
|
+
minLength: 1
|
|
10119
|
+
},
|
|
10120
|
+
origin: {
|
|
10121
|
+
type: "string",
|
|
10122
|
+
minLength: 1
|
|
10123
|
+
},
|
|
10124
|
+
generated: {
|
|
10125
|
+
type: "boolean"
|
|
10126
|
+
},
|
|
10127
|
+
checksum: {
|
|
10128
|
+
type: "string"
|
|
10129
|
+
},
|
|
10130
|
+
updated_at: {
|
|
10131
|
+
type: "string",
|
|
10132
|
+
format: "date-time"
|
|
10133
|
+
}
|
|
10134
|
+
}
|
|
10135
|
+
},
|
|
10136
|
+
title: {
|
|
10137
|
+
type: "string"
|
|
10138
|
+
},
|
|
10139
|
+
name: {
|
|
10140
|
+
type: "string"
|
|
10141
|
+
},
|
|
10142
|
+
summary: {
|
|
10143
|
+
type: "string"
|
|
10144
|
+
},
|
|
10145
|
+
text: {
|
|
10146
|
+
type: "string"
|
|
10147
|
+
},
|
|
10148
|
+
search: {
|
|
10149
|
+
type: "object",
|
|
10150
|
+
additionalProperties: false,
|
|
10151
|
+
required: [
|
|
10152
|
+
"title",
|
|
10153
|
+
"body",
|
|
10154
|
+
"triggers",
|
|
10155
|
+
"aliases",
|
|
10156
|
+
"tags",
|
|
10157
|
+
"frontmatter"
|
|
10158
|
+
],
|
|
10159
|
+
properties: {
|
|
10160
|
+
title: {
|
|
10161
|
+
type: "string"
|
|
10162
|
+
},
|
|
10163
|
+
name: {
|
|
10164
|
+
type: "string"
|
|
10165
|
+
},
|
|
10166
|
+
summary: {
|
|
10167
|
+
type: "string"
|
|
10168
|
+
},
|
|
10169
|
+
body: {
|
|
10170
|
+
type: "string"
|
|
10171
|
+
},
|
|
10172
|
+
triggers: {
|
|
10173
|
+
$ref: "#/$defs/stringArray"
|
|
10174
|
+
},
|
|
10175
|
+
aliases: {
|
|
10176
|
+
$ref: "#/$defs/stringArray"
|
|
10177
|
+
},
|
|
10178
|
+
capability: {
|
|
10179
|
+
type: "string"
|
|
10180
|
+
},
|
|
10181
|
+
tags: {
|
|
10182
|
+
$ref: "#/$defs/stringArray"
|
|
10183
|
+
},
|
|
10184
|
+
phase: {
|
|
10185
|
+
type: "string"
|
|
10186
|
+
},
|
|
10187
|
+
type: {
|
|
10188
|
+
type: "string"
|
|
10189
|
+
},
|
|
10190
|
+
frontmatter: {
|
|
10191
|
+
type: "object"
|
|
10192
|
+
}
|
|
10193
|
+
}
|
|
10194
|
+
},
|
|
10195
|
+
facets: {
|
|
10196
|
+
type: "object",
|
|
10197
|
+
additionalProperties: {
|
|
10198
|
+
$ref: "#/$defs/stringArray"
|
|
10199
|
+
}
|
|
10200
|
+
},
|
|
10201
|
+
tags: {
|
|
10202
|
+
$ref: "#/$defs/stringArray"
|
|
10203
|
+
},
|
|
10204
|
+
concepts: {
|
|
10205
|
+
$ref: "#/$defs/stringArray"
|
|
10206
|
+
},
|
|
10207
|
+
relationships: {
|
|
10208
|
+
type: "array",
|
|
10209
|
+
items: {
|
|
10210
|
+
type: "object",
|
|
10211
|
+
additionalProperties: false,
|
|
10212
|
+
required: ["type", "target_id"],
|
|
10213
|
+
properties: {
|
|
10214
|
+
type: {
|
|
10215
|
+
type: "string",
|
|
10216
|
+
minLength: 1
|
|
10217
|
+
},
|
|
10218
|
+
target_id: {
|
|
10219
|
+
type: "string",
|
|
10220
|
+
minLength: 1
|
|
10221
|
+
},
|
|
10222
|
+
source_path: {
|
|
10223
|
+
type: "string"
|
|
10224
|
+
},
|
|
10225
|
+
target_path: {
|
|
10226
|
+
type: "string"
|
|
10227
|
+
},
|
|
10228
|
+
direction: {
|
|
10229
|
+
enum: ["upstream", "downstream", "related"]
|
|
10230
|
+
},
|
|
10231
|
+
label: {
|
|
10232
|
+
type: "string"
|
|
10233
|
+
},
|
|
10234
|
+
confidence: {
|
|
10235
|
+
type: "number"
|
|
10236
|
+
},
|
|
10237
|
+
privacy: {
|
|
10238
|
+
$ref: "#/$defs/privacy"
|
|
10239
|
+
},
|
|
10240
|
+
metadata: {
|
|
10241
|
+
type: "object"
|
|
10242
|
+
}
|
|
10243
|
+
}
|
|
10244
|
+
}
|
|
10245
|
+
},
|
|
10246
|
+
provenance: {
|
|
10247
|
+
type: "array",
|
|
10248
|
+
minItems: 1,
|
|
10249
|
+
items: {
|
|
10250
|
+
type: "object",
|
|
10251
|
+
additionalProperties: false,
|
|
10252
|
+
required: ["field", "source", "path", "confidence", "privacy"],
|
|
10253
|
+
properties: {
|
|
10254
|
+
field: {
|
|
10255
|
+
type: "string",
|
|
10256
|
+
minLength: 1
|
|
10257
|
+
},
|
|
10258
|
+
source: {
|
|
10259
|
+
type: "string",
|
|
10260
|
+
minLength: 1
|
|
10261
|
+
},
|
|
10262
|
+
path: {
|
|
10263
|
+
type: "string",
|
|
10264
|
+
minLength: 1
|
|
10265
|
+
},
|
|
10266
|
+
confidence: {
|
|
10267
|
+
enum: ["source", "candidate", "reviewed", "rejected"]
|
|
10268
|
+
},
|
|
10269
|
+
privacy: {
|
|
10270
|
+
$ref: "#/$defs/privacy"
|
|
10271
|
+
}
|
|
10272
|
+
}
|
|
10273
|
+
}
|
|
10274
|
+
},
|
|
10275
|
+
privacy: {
|
|
10276
|
+
type: "object",
|
|
10277
|
+
additionalProperties: false,
|
|
10278
|
+
required: ["classification", "pii"],
|
|
10279
|
+
properties: {
|
|
10280
|
+
classification: {
|
|
10281
|
+
$ref: "#/$defs/privacy"
|
|
10282
|
+
},
|
|
10283
|
+
pii: {
|
|
10284
|
+
type: "boolean"
|
|
10285
|
+
},
|
|
10286
|
+
locality: {
|
|
10287
|
+
enum: ["project", "framework", "external"]
|
|
10288
|
+
}
|
|
10289
|
+
}
|
|
10290
|
+
},
|
|
10291
|
+
chunks: {
|
|
10292
|
+
type: "array",
|
|
10293
|
+
items: {
|
|
10294
|
+
type: "object",
|
|
10295
|
+
additionalProperties: false,
|
|
10296
|
+
properties: {
|
|
10297
|
+
id: {
|
|
10298
|
+
type: "string",
|
|
10299
|
+
minLength: 1
|
|
10300
|
+
},
|
|
10301
|
+
text: {
|
|
10302
|
+
type: "string"
|
|
10303
|
+
},
|
|
10304
|
+
body: {
|
|
10305
|
+
type: "string"
|
|
10306
|
+
},
|
|
10307
|
+
summary: {
|
|
10308
|
+
type: "string"
|
|
10309
|
+
},
|
|
10310
|
+
source_path: {
|
|
10311
|
+
type: "string"
|
|
10312
|
+
},
|
|
10313
|
+
metadata: {
|
|
10314
|
+
type: "object"
|
|
10315
|
+
},
|
|
10316
|
+
checksum: {
|
|
10317
|
+
type: "string"
|
|
10318
|
+
}
|
|
10319
|
+
}
|
|
10320
|
+
}
|
|
10321
|
+
},
|
|
10322
|
+
embeddings: {
|
|
10323
|
+
type: "array",
|
|
10324
|
+
items: {
|
|
10325
|
+
type: "object",
|
|
10326
|
+
additionalProperties: false,
|
|
10327
|
+
properties: {
|
|
10328
|
+
id: {
|
|
10329
|
+
type: "string"
|
|
10330
|
+
},
|
|
10331
|
+
model: {
|
|
10332
|
+
type: "string"
|
|
10333
|
+
},
|
|
10334
|
+
embedding: {
|
|
10335
|
+
$ref: "#/$defs/numberArray"
|
|
10336
|
+
},
|
|
10337
|
+
vector: {
|
|
10338
|
+
$ref: "#/$defs/numberArray"
|
|
10339
|
+
},
|
|
10340
|
+
granularity: {
|
|
10341
|
+
type: "string"
|
|
10342
|
+
},
|
|
10343
|
+
source_path: {
|
|
10344
|
+
type: "string"
|
|
10345
|
+
},
|
|
10346
|
+
metadata: {
|
|
10347
|
+
type: "object"
|
|
10348
|
+
},
|
|
10349
|
+
chunk_id: {
|
|
10350
|
+
type: "string"
|
|
10351
|
+
},
|
|
10352
|
+
vector_ref: {
|
|
10353
|
+
type: "string"
|
|
10354
|
+
},
|
|
10355
|
+
input_hash: {
|
|
10356
|
+
type: "string"
|
|
10357
|
+
}
|
|
10358
|
+
}
|
|
10359
|
+
}
|
|
10360
|
+
},
|
|
10361
|
+
compatibility: {
|
|
10362
|
+
type: "object"
|
|
10363
|
+
},
|
|
10364
|
+
skos_concepts: {
|
|
10365
|
+
type: "array",
|
|
10366
|
+
items: {
|
|
10367
|
+
type: "object",
|
|
10368
|
+
additionalProperties: false,
|
|
10369
|
+
required: ["id", "prefLabel"],
|
|
10370
|
+
properties: {
|
|
10371
|
+
id: {
|
|
10372
|
+
type: "string",
|
|
10373
|
+
minLength: 1
|
|
10374
|
+
},
|
|
10375
|
+
prefLabel: {
|
|
10376
|
+
type: "string",
|
|
10377
|
+
minLength: 1
|
|
10378
|
+
},
|
|
10379
|
+
definition: {
|
|
10380
|
+
type: "string"
|
|
10381
|
+
},
|
|
10382
|
+
scheme: {
|
|
10383
|
+
type: "string"
|
|
10384
|
+
},
|
|
10385
|
+
notation: {
|
|
10386
|
+
type: "string"
|
|
10387
|
+
},
|
|
10388
|
+
uri: {
|
|
10389
|
+
type: "string"
|
|
10390
|
+
},
|
|
10391
|
+
altLabels: {
|
|
10392
|
+
$ref: "#/$defs/stringArray"
|
|
10393
|
+
},
|
|
10394
|
+
metadata: {
|
|
10395
|
+
type: "object"
|
|
10396
|
+
}
|
|
10397
|
+
}
|
|
10398
|
+
}
|
|
10399
|
+
},
|
|
10400
|
+
skos_relations: {
|
|
10401
|
+
type: "array",
|
|
10402
|
+
items: {
|
|
10403
|
+
type: "object",
|
|
10404
|
+
additionalProperties: false,
|
|
10405
|
+
required: ["type", "source_id", "target_id"],
|
|
10406
|
+
properties: {
|
|
10407
|
+
type: {
|
|
10408
|
+
type: "string",
|
|
10409
|
+
minLength: 1
|
|
10410
|
+
},
|
|
10411
|
+
source_id: {
|
|
10412
|
+
type: "string",
|
|
10413
|
+
minLength: 1
|
|
10414
|
+
},
|
|
10415
|
+
target_id: {
|
|
10416
|
+
type: "string",
|
|
10417
|
+
minLength: 1
|
|
10418
|
+
},
|
|
10419
|
+
source_path: {
|
|
10420
|
+
type: "string"
|
|
10421
|
+
},
|
|
10422
|
+
metadata: {
|
|
10423
|
+
type: "object"
|
|
10424
|
+
}
|
|
10425
|
+
}
|
|
10426
|
+
}
|
|
10427
|
+
},
|
|
10428
|
+
provenance_events: {
|
|
10429
|
+
type: "array",
|
|
10430
|
+
items: {
|
|
10431
|
+
type: "object",
|
|
10432
|
+
additionalProperties: false,
|
|
10433
|
+
required: ["activity"],
|
|
10434
|
+
properties: {
|
|
10435
|
+
id: {
|
|
10436
|
+
type: "string"
|
|
10437
|
+
},
|
|
10438
|
+
activity: {
|
|
10439
|
+
type: "string",
|
|
10440
|
+
minLength: 1
|
|
10441
|
+
},
|
|
10442
|
+
agent: {
|
|
10443
|
+
type: "string"
|
|
10444
|
+
},
|
|
10445
|
+
started_at: {
|
|
10446
|
+
type: "string",
|
|
10447
|
+
format: "date-time"
|
|
10448
|
+
},
|
|
10449
|
+
ended_at: {
|
|
10450
|
+
type: "string",
|
|
10451
|
+
format: "date-time"
|
|
10452
|
+
},
|
|
10453
|
+
source: {
|
|
10454
|
+
type: "string"
|
|
10455
|
+
},
|
|
10456
|
+
path: {
|
|
10457
|
+
type: "string"
|
|
10458
|
+
},
|
|
10459
|
+
confidence: {
|
|
10460
|
+
enum: ["source", "candidate", "reviewed", "rejected"]
|
|
10461
|
+
},
|
|
10462
|
+
privacy: {
|
|
10463
|
+
$ref: "#/$defs/privacy"
|
|
10464
|
+
},
|
|
10465
|
+
attributes: {
|
|
10466
|
+
type: "object"
|
|
10467
|
+
}
|
|
10468
|
+
}
|
|
10469
|
+
}
|
|
10470
|
+
},
|
|
10471
|
+
updated_at: {
|
|
10472
|
+
type: "string",
|
|
10473
|
+
format: "date-time"
|
|
10474
|
+
}
|
|
10475
|
+
}
|
|
10476
|
+
}
|
|
10477
|
+
}
|
|
10478
|
+
};
|
|
10479
|
+
|
|
10480
|
+
// src/aiwg-index-schema.ts
|
|
10481
|
+
var PROJECTED_FIELDS = [
|
|
10482
|
+
"schema_version",
|
|
10483
|
+
"id",
|
|
10484
|
+
"type",
|
|
10485
|
+
"title",
|
|
10486
|
+
"text",
|
|
10487
|
+
"facets",
|
|
10488
|
+
"tags",
|
|
10489
|
+
"concepts",
|
|
10490
|
+
"privacy"
|
|
10491
|
+
];
|
|
10492
|
+
var ajvInstance2;
|
|
10493
|
+
var exportValidator;
|
|
10494
|
+
var projectedRecordValidator;
|
|
10495
|
+
function getAiwgFortemiIndexExportSchema() {
|
|
10496
|
+
return aiwg_fortemi_index_export_schema_default;
|
|
10497
|
+
}
|
|
10498
|
+
function getAjv2() {
|
|
10499
|
+
if (!ajvInstance2) {
|
|
10500
|
+
ajvInstance2 = new Ajv2020({
|
|
10501
|
+
allErrors: true,
|
|
10502
|
+
strict: false,
|
|
10503
|
+
validateFormats: false
|
|
10504
|
+
});
|
|
10505
|
+
ajvInstance2.addSchema(aiwg_fortemi_index_export_schema_default);
|
|
10506
|
+
}
|
|
10507
|
+
return ajvInstance2;
|
|
10508
|
+
}
|
|
10509
|
+
function formatErrors2(errors) {
|
|
10510
|
+
return (errors ?? []).map((error) => {
|
|
10511
|
+
const path = error.instancePath || "(root)";
|
|
10512
|
+
return `${path} ${error.message ?? "is invalid"}`;
|
|
10513
|
+
});
|
|
10514
|
+
}
|
|
10515
|
+
function getExportValidator() {
|
|
10516
|
+
exportValidator ??= getAjv2().getSchema(aiwg_fortemi_index_export_schema_default.$id) ?? getAjv2().compile(aiwg_fortemi_index_export_schema_default);
|
|
10517
|
+
return exportValidator;
|
|
10518
|
+
}
|
|
10519
|
+
function getProjectedRecordValidator() {
|
|
10520
|
+
if (!projectedRecordValidator) {
|
|
10521
|
+
const properties = Object.fromEntries(Object.keys(aiwg_fortemi_index_export_schema_default.$defs.record.properties).map((field) => [
|
|
10522
|
+
field,
|
|
10523
|
+
{ $ref: `${aiwg_fortemi_index_export_schema_default.$id}#/$defs/record/properties/${field}` }
|
|
10524
|
+
]));
|
|
10525
|
+
projectedRecordValidator = getAjv2().compile({
|
|
10526
|
+
type: "object",
|
|
10527
|
+
required: PROJECTED_FIELDS,
|
|
10528
|
+
properties,
|
|
10529
|
+
additionalProperties: true,
|
|
10530
|
+
allOf: [
|
|
10531
|
+
{
|
|
10532
|
+
if: {
|
|
10533
|
+
properties: {
|
|
10534
|
+
schema_version: { const: "aiwg.fortemi.index.record.v1" }
|
|
10535
|
+
}
|
|
10536
|
+
},
|
|
10537
|
+
then: { $ref: `${aiwg_fortemi_index_export_schema_default.$id}#/$defs/v1RecordCompatibility` }
|
|
10538
|
+
}
|
|
10539
|
+
]
|
|
10540
|
+
});
|
|
10541
|
+
}
|
|
10542
|
+
return projectedRecordValidator;
|
|
10543
|
+
}
|
|
10544
|
+
function validateAiwgFortemiIndexExportSchema(value) {
|
|
10545
|
+
const validate = getExportValidator();
|
|
10546
|
+
const schemaValue = value && typeof value === "object" && Array.isArray(value.items) ? {
|
|
10547
|
+
...value,
|
|
10548
|
+
items: value.items.map((item) => {
|
|
10549
|
+
if (!item || typeof item !== "object" || Array.isArray(item)) return item;
|
|
10550
|
+
const schemaRecord = { ...item };
|
|
10551
|
+
Reflect.deleteProperty(schemaRecord, "binary_sources");
|
|
10552
|
+
return schemaRecord;
|
|
10553
|
+
})
|
|
10554
|
+
} : value;
|
|
10555
|
+
const valid = validate(schemaValue);
|
|
10556
|
+
return { valid, errors: formatErrors2(validate.errors) };
|
|
10557
|
+
}
|
|
10558
|
+
function validateAiwgFortemiProjectedRecordSchema(value) {
|
|
10559
|
+
const validate = getProjectedRecordValidator();
|
|
10560
|
+
const valid = validate(value);
|
|
10561
|
+
return { valid, errors: formatErrors2(validate.errors) };
|
|
10562
|
+
}
|
|
10563
|
+
|
|
8219
10564
|
// src/aiwg-index.ts
|
|
8220
10565
|
var AIWG_SCAN_REQUIRED_FIELDS = [
|
|
8221
10566
|
"schema_version",
|
|
@@ -8230,7 +10575,7 @@ var AIWG_SCAN_REQUIRED_FIELDS = [
|
|
|
8230
10575
|
];
|
|
8231
10576
|
function isPrivacyExcluded(record, options) {
|
|
8232
10577
|
const privacy = record.privacy;
|
|
8233
|
-
if (!privacy) return
|
|
10578
|
+
if (!privacy || !isPrivacyClassification(privacy.classification) || typeof privacy.pii !== "boolean") return true;
|
|
8234
10579
|
if (privacy.classification === "private" && !options?.includePrivate) return true;
|
|
8235
10580
|
if (privacy.pii && !options?.includePii) return true;
|
|
8236
10581
|
return false;
|
|
@@ -8299,6 +10644,10 @@ function validateOptionalRichMetadata(item, index, errors) {
|
|
|
8299
10644
|
errors.push("items[" + index + "].skos_concepts must be an array when present");
|
|
8300
10645
|
} else {
|
|
8301
10646
|
for (const [conceptIndex, concept] of item.skos_concepts.entries()) {
|
|
10647
|
+
if (!isPlainRecord(concept)) {
|
|
10648
|
+
errors.push("items[" + index + "].skos_concepts[" + conceptIndex + "] must be an object");
|
|
10649
|
+
continue;
|
|
10650
|
+
}
|
|
8302
10651
|
if (!hasString(concept.id)) errors.push("items[" + index + "].skos_concepts[" + conceptIndex + "].id is required");
|
|
8303
10652
|
if (!hasString(concept.prefLabel)) errors.push("items[" + index + "].skos_concepts[" + conceptIndex + "].prefLabel is required");
|
|
8304
10653
|
if (!isOptionalStringArray(concept.altLabels)) errors.push("items[" + index + "].skos_concepts[" + conceptIndex + "].altLabels must be a string array");
|
|
@@ -8313,6 +10662,10 @@ function validateOptionalRichMetadata(item, index, errors) {
|
|
|
8313
10662
|
errors.push("items[" + index + "].skos_relations must be an array when present");
|
|
8314
10663
|
} else {
|
|
8315
10664
|
for (const [relationIndex, relation] of item.skos_relations.entries()) {
|
|
10665
|
+
if (!isPlainRecord(relation)) {
|
|
10666
|
+
errors.push("items[" + index + "].skos_relations[" + relationIndex + "] must be an object");
|
|
10667
|
+
continue;
|
|
10668
|
+
}
|
|
8316
10669
|
if (!hasString(relation.type)) errors.push("items[" + index + "].skos_relations[" + relationIndex + "].type is required");
|
|
8317
10670
|
if (!hasString(relation.source_id)) errors.push("items[" + index + "].skos_relations[" + relationIndex + "].source_id is required");
|
|
8318
10671
|
if (!hasString(relation.target_id)) errors.push("items[" + index + "].skos_relations[" + relationIndex + "].target_id is required");
|
|
@@ -8327,6 +10680,10 @@ function validateOptionalRichMetadata(item, index, errors) {
|
|
|
8327
10680
|
errors.push("items[" + index + "].provenance_events must be an array when present");
|
|
8328
10681
|
} else {
|
|
8329
10682
|
for (const [eventIndex, event] of item.provenance_events.entries()) {
|
|
10683
|
+
if (!isPlainRecord(event)) {
|
|
10684
|
+
errors.push("items[" + index + "].provenance_events[" + eventIndex + "] must be an object");
|
|
10685
|
+
continue;
|
|
10686
|
+
}
|
|
8330
10687
|
if (!hasString(event.activity)) errors.push("items[" + index + "].provenance_events[" + eventIndex + "].activity is required");
|
|
8331
10688
|
if (event.attributes !== void 0 && !isPlainRecord(event.attributes)) {
|
|
8332
10689
|
errors.push("items[" + index + "].provenance_events[" + eventIndex + "].attributes must be an object");
|
|
@@ -8336,6 +10693,10 @@ function validateOptionalRichMetadata(item, index, errors) {
|
|
|
8336
10693
|
}
|
|
8337
10694
|
if (Array.isArray(item.relationships)) {
|
|
8338
10695
|
for (const [relationshipIndex, relationship] of item.relationships.entries()) {
|
|
10696
|
+
if (!isPlainRecord(relationship)) {
|
|
10697
|
+
errors.push("items[" + index + "].relationships[" + relationshipIndex + "] must be an object");
|
|
10698
|
+
continue;
|
|
10699
|
+
}
|
|
8339
10700
|
if (relationship.metadata !== void 0 && !isPlainRecord(relationship.metadata)) {
|
|
8340
10701
|
errors.push("items[" + index + "].relationships[" + relationshipIndex + "].metadata must be an object");
|
|
8341
10702
|
}
|
|
@@ -8364,7 +10725,10 @@ function validateOptionalRichMetadata(item, index, errors) {
|
|
|
8364
10725
|
errors.push("items[" + index + "].chunks must be an array when present");
|
|
8365
10726
|
} else {
|
|
8366
10727
|
for (const [chunkIndex, chunk] of item.chunks.entries()) {
|
|
8367
|
-
if (!isPlainRecord(chunk))
|
|
10728
|
+
if (!isPlainRecord(chunk)) {
|
|
10729
|
+
errors.push("items[" + index + "].chunks[" + chunkIndex + "] must be an object");
|
|
10730
|
+
continue;
|
|
10731
|
+
}
|
|
8368
10732
|
if (chunk.metadata !== void 0 && !isPlainRecord(chunk.metadata)) {
|
|
8369
10733
|
errors.push("items[" + index + "].chunks[" + chunkIndex + "].metadata must be an object");
|
|
8370
10734
|
}
|
|
@@ -8376,9 +10740,12 @@ function validateOptionalRichMetadata(item, index, errors) {
|
|
|
8376
10740
|
errors.push("items[" + index + "].embeddings must be an array when present");
|
|
8377
10741
|
} else {
|
|
8378
10742
|
for (const [embeddingIndex, embedding] of item.embeddings.entries()) {
|
|
8379
|
-
if (!isPlainRecord(embedding))
|
|
8380
|
-
|
|
8381
|
-
|
|
10743
|
+
if (!isPlainRecord(embedding)) {
|
|
10744
|
+
errors.push("items[" + index + "].embeddings[" + embeddingIndex + "] must be an object");
|
|
10745
|
+
continue;
|
|
10746
|
+
}
|
|
10747
|
+
const vector = embedding.embedding ?? embedding.vector;
|
|
10748
|
+
if (vector !== void 0 && (!Array.isArray(vector) || !vector.every((entry) => typeof entry === "number"))) {
|
|
8382
10749
|
errors.push("items[" + index + "].embeddings[" + embeddingIndex + "].embedding/vector must be a number array");
|
|
8383
10750
|
}
|
|
8384
10751
|
if (embedding.metadata !== void 0 && !isPlainRecord(embedding.metadata)) {
|
|
@@ -8443,9 +10810,10 @@ function forbidV2FieldsOnV1Record(item, index, errors) {
|
|
|
8443
10810
|
}
|
|
8444
10811
|
}
|
|
8445
10812
|
function validateAiwgFortemiIndexExport(value) {
|
|
8446
|
-
const errors =
|
|
8447
|
-
const counts =
|
|
8448
|
-
const data = value;
|
|
10813
|
+
const errors = validateAiwgFortemiIndexExportSchema(value).errors;
|
|
10814
|
+
const counts = /* @__PURE__ */ Object.create(null);
|
|
10815
|
+
const data = isPlainRecord(value) ? value : {};
|
|
10816
|
+
if (!isPlainRecord(value)) errors.push("index export must be an object");
|
|
8449
10817
|
if (!isSupportedIndexSchemaVersion(data?.schema_version)) {
|
|
8450
10818
|
errors.push("schema_version must be aiwg.fortemi.index.export.v1 or aiwg.fortemi.index.export.v2");
|
|
8451
10819
|
}
|
|
@@ -8469,7 +10837,12 @@ function validateAiwgFortemiIndexExport(value) {
|
|
|
8469
10837
|
}
|
|
8470
10838
|
const ids = /* @__PURE__ */ new Set();
|
|
8471
10839
|
let previousId = "";
|
|
8472
|
-
|
|
10840
|
+
const items = Array.isArray(data.items) ? data.items : [];
|
|
10841
|
+
for (const [index, item] of items.entries()) {
|
|
10842
|
+
if (!isPlainRecord(item)) {
|
|
10843
|
+
errors.push("items[" + index + "] must be an object");
|
|
10844
|
+
continue;
|
|
10845
|
+
}
|
|
8473
10846
|
for (const field of REQUIRED_RECORD_FIELDS) {
|
|
8474
10847
|
if (!(field in item)) errors.push("items[" + index + "]." + field + " is required");
|
|
8475
10848
|
}
|
|
@@ -8479,7 +10852,7 @@ function validateAiwgFortemiIndexExport(value) {
|
|
|
8479
10852
|
if (!hasString(item.id)) errors.push("items[" + index + "].id is required");
|
|
8480
10853
|
if (hasString(item.id) && ids.has(item.id)) errors.push("duplicate id: " + item.id);
|
|
8481
10854
|
if (hasString(item.id)) ids.add(item.id);
|
|
8482
|
-
if (previousId && hasString(item.id) && previousId
|
|
10855
|
+
if (previousId && hasString(item.id) && previousId > item.id) {
|
|
8483
10856
|
errors.push("items must be sorted by id: " + previousId + " before " + item.id);
|
|
8484
10857
|
}
|
|
8485
10858
|
if (hasString(item.id)) previousId = item.id;
|
|
@@ -8520,7 +10893,8 @@ function assertAiwgFortemiIndexExport(value) {
|
|
|
8520
10893
|
}
|
|
8521
10894
|
function validateAiwgFortemiChunkManifest(value) {
|
|
8522
10895
|
const errors = [];
|
|
8523
|
-
const data = value;
|
|
10896
|
+
const data = isPlainRecord(value) ? value : {};
|
|
10897
|
+
if (!isPlainRecord(value)) errors.push("chunk manifest must be an object");
|
|
8524
10898
|
if (data?.schema_version !== "aiwg.fortemi.index.chunk-manifest.v1") {
|
|
8525
10899
|
errors.push("schema_version must be aiwg.fortemi.index.chunk-manifest.v1");
|
|
8526
10900
|
}
|
|
@@ -8545,7 +10919,9 @@ function validateAiwgFortemiChunkManifest(value) {
|
|
|
8545
10919
|
}
|
|
8546
10920
|
}
|
|
8547
10921
|
}
|
|
8548
|
-
if (data.detail !== void 0) {
|
|
10922
|
+
if (data.detail !== void 0 && !isPlainRecord(data.detail)) {
|
|
10923
|
+
errors.push("detail must be an object");
|
|
10924
|
+
} else if (data.detail !== void 0) {
|
|
8549
10925
|
if (!hasString(data.detail.href)) errors.push("detail.href is required");
|
|
8550
10926
|
else if (!data.detail.href.includes("{id}")) errors.push("detail.href must contain the {id} placeholder");
|
|
8551
10927
|
if (data.detail.encoding !== void 0 && data.detail.encoding !== "uri" && data.detail.encoding !== "base64url") {
|
|
@@ -8556,6 +10932,10 @@ function validateAiwgFortemiChunkManifest(value) {
|
|
|
8556
10932
|
let expectedOffset = 0;
|
|
8557
10933
|
const parts = Array.isArray(data?.parts) ? data.parts : [];
|
|
8558
10934
|
for (const [index, part] of parts.entries()) {
|
|
10935
|
+
if (!isPlainRecord(part)) {
|
|
10936
|
+
errors.push("parts[" + index + "] must be an object");
|
|
10937
|
+
continue;
|
|
10938
|
+
}
|
|
8559
10939
|
if (!hasString(part.href)) errors.push("parts[" + index + "].href is required");
|
|
8560
10940
|
if (!hasNonNegativeInteger(part.offset)) errors.push("parts[" + index + "].offset must be a non-negative integer");
|
|
8561
10941
|
if (!hasNonNegativeInteger(part.count)) errors.push("parts[" + index + "].count must be a non-negative integer");
|
|
@@ -8576,18 +10956,28 @@ function assertAiwgFortemiChunkManifest(value) {
|
|
|
8576
10956
|
}
|
|
8577
10957
|
return value;
|
|
8578
10958
|
}
|
|
8579
|
-
function validateProjectedRecords(items) {
|
|
10959
|
+
function validateProjectedRecords(items, sourceSchemaVersion) {
|
|
8580
10960
|
const errors = [];
|
|
8581
10961
|
const ids = /* @__PURE__ */ new Set();
|
|
8582
10962
|
let previousId = "";
|
|
8583
10963
|
for (const [index, item] of items.entries()) {
|
|
10964
|
+
if (!isPlainRecord(item)) {
|
|
10965
|
+
errors.push("items[" + index + "] must be an object");
|
|
10966
|
+
continue;
|
|
10967
|
+
}
|
|
10968
|
+
const schemaValidation = validateAiwgFortemiProjectedRecordSchema(item);
|
|
10969
|
+
errors.push(...schemaValidation.errors.map((error) => `items[${index}]${error}`));
|
|
10970
|
+
const expectedRecordVersion = sourceSchemaVersion === "aiwg.fortemi.index.export.v2" ? "aiwg.fortemi.index.record.v2" : "aiwg.fortemi.index.record.v1";
|
|
10971
|
+
if (item.schema_version !== expectedRecordVersion) {
|
|
10972
|
+
errors.push(`items[${index}].schema_version must match ${sourceSchemaVersion}`);
|
|
10973
|
+
}
|
|
8584
10974
|
if (!isSupportedRecordSchemaVersion(item.schema_version)) {
|
|
8585
10975
|
errors.push("items[" + index + "].schema_version must be aiwg.fortemi.index.record.v1 or aiwg.fortemi.index.record.v2");
|
|
8586
10976
|
}
|
|
8587
10977
|
if (!hasString(item.id)) errors.push("items[" + index + "].id is required");
|
|
8588
10978
|
if (hasString(item.id) && ids.has(item.id)) errors.push("duplicate id: " + item.id);
|
|
8589
10979
|
if (hasString(item.id)) ids.add(item.id);
|
|
8590
|
-
if (previousId && hasString(item.id) && previousId
|
|
10980
|
+
if (previousId && hasString(item.id) && previousId > item.id) {
|
|
8591
10981
|
errors.push("items must be sorted by id: " + previousId + " before " + item.id);
|
|
8592
10982
|
}
|
|
8593
10983
|
if (hasString(item.id)) previousId = item.id;
|
|
@@ -8603,15 +10993,13 @@ function validateProjectedRecords(items) {
|
|
|
8603
10993
|
}
|
|
8604
10994
|
if (!Array.isArray(item.tags)) errors.push("items[" + index + "].tags must be an array");
|
|
8605
10995
|
if (!Array.isArray(item.concepts)) errors.push("items[" + index + "].concepts must be an array");
|
|
8606
|
-
if (!item.privacy || !hasString(item.privacy.classification)) {
|
|
8607
|
-
errors.push("items[" + index + "].privacy.classification is required");
|
|
8608
|
-
}
|
|
8609
10996
|
}
|
|
8610
10997
|
return errors;
|
|
8611
10998
|
}
|
|
8612
10999
|
function validateAiwgFortemiChunkPart(value, partRef, manifest) {
|
|
8613
11000
|
const errors = [];
|
|
8614
|
-
const data = value;
|
|
11001
|
+
const data = isPlainRecord(value) ? value : {};
|
|
11002
|
+
if (!isPlainRecord(value)) errors.push("chunk part must be an object");
|
|
8615
11003
|
if (data?.schema_version !== "aiwg.fortemi.index.chunk.v1") {
|
|
8616
11004
|
errors.push("schema_version must be aiwg.fortemi.index.chunk.v1");
|
|
8617
11005
|
}
|
|
@@ -8628,12 +11016,16 @@ function validateAiwgFortemiChunkPart(value, partRef, manifest) {
|
|
|
8628
11016
|
}
|
|
8629
11017
|
if (Array.isArray(data?.items)) {
|
|
8630
11018
|
if (manifest?.projection) {
|
|
8631
|
-
errors.push(...validateProjectedRecords(
|
|
11019
|
+
errors.push(...validateProjectedRecords(
|
|
11020
|
+
data.items,
|
|
11021
|
+
manifest.source_export_schema_version ?? "aiwg.fortemi.index.export.v1"
|
|
11022
|
+
).map((error) => "items." + error));
|
|
8632
11023
|
} else {
|
|
8633
11024
|
const validation = validateAiwgFortemiIndexExport({
|
|
8634
|
-
schema_version: "aiwg.fortemi.index.export.v1",
|
|
11025
|
+
schema_version: manifest?.source_export_schema_version ?? "aiwg.fortemi.index.export.v1",
|
|
8635
11026
|
generated_at: manifest?.generated_at ?? "1970-01-01T00:00:00.000Z",
|
|
8636
11027
|
source: manifest?.source ?? { repo: "chunk", privacy: "public" },
|
|
11028
|
+
...(manifest?.source_export_schema_version ?? "aiwg.fortemi.index.export.v1") === "aiwg.fortemi.index.export.v2" ? { compatibility: { previous_schema_version: "aiwg.fortemi.index.export.v1", strategy: "supported" } } : {},
|
|
8637
11029
|
items: data.items
|
|
8638
11030
|
});
|
|
8639
11031
|
errors.push(...validation.errors.map((error) => "items." + error));
|
|
@@ -8720,15 +11112,17 @@ function recordTitle(item) {
|
|
|
8720
11112
|
}
|
|
8721
11113
|
function recordText(item) {
|
|
8722
11114
|
const base = item.text ?? item.search?.body ?? item.search?.summary ?? item.chunks?.map((chunk) => chunk.text ?? chunk.body ?? chunk.summary ?? "").filter(Boolean).join("\n") ?? "";
|
|
8723
|
-
const extractedText =
|
|
11115
|
+
const extractedText = binarySourceText(item);
|
|
8724
11116
|
return [base, extractedText].filter(Boolean).join("\n");
|
|
8725
11117
|
}
|
|
11118
|
+
function binarySourceText(item) {
|
|
11119
|
+
return "binary_sources" in item ? item.binary_sources?.map((source) => source.extracted_text).filter(Boolean).join("\n") ?? "" : "";
|
|
11120
|
+
}
|
|
8726
11121
|
function defaultEmbeddingInput(record, granularity) {
|
|
8727
11122
|
const title = recordTitle(record);
|
|
8728
11123
|
const text = recordText(record);
|
|
8729
11124
|
if (granularity === "title-summary") {
|
|
8730
|
-
|
|
8731
|
-
return [title, record.search?.summary ?? "", extractedText].filter(Boolean).join("\n");
|
|
11125
|
+
return [title, record.search?.summary ?? "", binarySourceText(record)].filter(Boolean).join("\n");
|
|
8732
11126
|
}
|
|
8733
11127
|
return [title, text].filter(Boolean).join("\n");
|
|
8734
11128
|
}
|
|
@@ -8796,6 +11190,7 @@ function recordSearchValues(item) {
|
|
|
8796
11190
|
return values.filter((value) => typeof value === "string" && value.length > 0);
|
|
8797
11191
|
}
|
|
8798
11192
|
function buildAiwgChunkedIndex(index, options = {}) {
|
|
11193
|
+
assertAiwgFortemiIndexExport(index);
|
|
8799
11194
|
const partSize = hasPositiveInteger(options.partSize) ? options.partSize : 500;
|
|
8800
11195
|
const projection = options.projection;
|
|
8801
11196
|
const idEncoding = options.idEncoding ?? "base64url";
|
|
@@ -8875,35 +11270,88 @@ function queryMatches(item, q) {
|
|
|
8875
11270
|
return matches;
|
|
8876
11271
|
}
|
|
8877
11272
|
var DISCOVERY_STOPWORDS = /* @__PURE__ */ new Set([
|
|
11273
|
+
"the",
|
|
8878
11274
|
"a",
|
|
8879
11275
|
"an",
|
|
8880
11276
|
"and",
|
|
8881
|
-
"
|
|
8882
|
-
"
|
|
11277
|
+
"or",
|
|
11278
|
+
"of",
|
|
8883
11279
|
"for",
|
|
8884
|
-
"
|
|
8885
|
-
"how",
|
|
8886
|
-
"i",
|
|
11280
|
+
"to",
|
|
8887
11281
|
"in",
|
|
11282
|
+
"on",
|
|
11283
|
+
"with",
|
|
11284
|
+
"into",
|
|
11285
|
+
"from",
|
|
8888
11286
|
"is",
|
|
11287
|
+
"are",
|
|
11288
|
+
"be",
|
|
11289
|
+
"i",
|
|
11290
|
+
"we",
|
|
11291
|
+
"my",
|
|
11292
|
+
"it",
|
|
11293
|
+
"you",
|
|
8889
11294
|
"me",
|
|
8890
|
-
"
|
|
8891
|
-
"
|
|
8892
|
-
"
|
|
11295
|
+
"us",
|
|
11296
|
+
"your",
|
|
11297
|
+
"our",
|
|
11298
|
+
"this",
|
|
11299
|
+
"that",
|
|
11300
|
+
"these",
|
|
11301
|
+
"those",
|
|
11302
|
+
"there",
|
|
11303
|
+
"here",
|
|
11304
|
+
"some",
|
|
11305
|
+
"any",
|
|
11306
|
+
"all",
|
|
11307
|
+
"also",
|
|
8893
11308
|
"please",
|
|
8894
|
-
"
|
|
8895
|
-
"
|
|
8896
|
-
"
|
|
8897
|
-
"
|
|
11309
|
+
"about",
|
|
11310
|
+
"how",
|
|
11311
|
+
"what",
|
|
11312
|
+
"which",
|
|
11313
|
+
"where",
|
|
11314
|
+
"when",
|
|
11315
|
+
"who",
|
|
11316
|
+
"why",
|
|
11317
|
+
"find",
|
|
11318
|
+
"give",
|
|
11319
|
+
"show",
|
|
11320
|
+
"need",
|
|
11321
|
+
"want",
|
|
11322
|
+
"looking",
|
|
11323
|
+
"look",
|
|
11324
|
+
"help",
|
|
11325
|
+
"do",
|
|
11326
|
+
"does",
|
|
11327
|
+
"did",
|
|
11328
|
+
"can",
|
|
11329
|
+
"could",
|
|
11330
|
+
"should",
|
|
11331
|
+
"would",
|
|
11332
|
+
"will",
|
|
11333
|
+
"handle",
|
|
11334
|
+
"handles",
|
|
11335
|
+
"handling",
|
|
11336
|
+
"aiwg",
|
|
11337
|
+
"skill",
|
|
11338
|
+
"skills",
|
|
11339
|
+
"agent",
|
|
11340
|
+
"agents",
|
|
11341
|
+
"command",
|
|
11342
|
+
"commands",
|
|
11343
|
+
"rule",
|
|
11344
|
+
"rules",
|
|
11345
|
+
"flow",
|
|
11346
|
+
"flows",
|
|
11347
|
+
"workflow",
|
|
11348
|
+
"workflows"
|
|
8898
11349
|
]);
|
|
8899
|
-
function
|
|
8900
|
-
return value.toLowerCase().replace(/[_
|
|
8901
|
-
}
|
|
8902
|
-
function canonicalDiscoveryName(value) {
|
|
8903
|
-
return normalizeDiscoveryText(value).replace(/[\s.-]+/g, "");
|
|
11350
|
+
function normalizeDiscoveryName(value) {
|
|
11351
|
+
return value.toLowerCase().replace(/[-_\s]+/g, " ").trim();
|
|
8904
11352
|
}
|
|
8905
11353
|
function discoveryTokens(value) {
|
|
8906
|
-
return
|
|
11354
|
+
return value.toLowerCase().split(/[^a-z0-9-]+/).filter((token) => token.length > 1 && !DISCOVERY_STOPWORDS.has(token));
|
|
8907
11355
|
}
|
|
8908
11356
|
function facetValues(item, names) {
|
|
8909
11357
|
return names.flatMap((name) => item.facets[name] ?? []);
|
|
@@ -8913,17 +11361,58 @@ function addDiscoveryMatch(matches, match) {
|
|
|
8913
11361
|
matches.push(match);
|
|
8914
11362
|
}
|
|
8915
11363
|
}
|
|
8916
|
-
function
|
|
8917
|
-
if (
|
|
8918
|
-
|
|
8919
|
-
|
|
8920
|
-
|
|
11364
|
+
function damerauLevenshteinAtMostOne(left, right) {
|
|
11365
|
+
if (left === right) return true;
|
|
11366
|
+
if (Math.abs(left.length - right.length) > 1) return false;
|
|
11367
|
+
if (left.length === right.length) {
|
|
11368
|
+
let firstDiff = -1;
|
|
11369
|
+
let diffCount = 0;
|
|
11370
|
+
for (let index = 0; index < left.length; index += 1) {
|
|
11371
|
+
if (left[index] !== right[index]) {
|
|
11372
|
+
if (firstDiff < 0) firstDiff = index;
|
|
11373
|
+
diffCount += 1;
|
|
11374
|
+
}
|
|
11375
|
+
}
|
|
11376
|
+
if (diffCount === 1) return true;
|
|
11377
|
+
return diffCount === 2 && firstDiff + 1 < left.length && left[firstDiff] === right[firstDiff + 1] && left[firstDiff + 1] === right[firstDiff];
|
|
11378
|
+
}
|
|
11379
|
+
const shorter = left.length < right.length ? left : right;
|
|
11380
|
+
const longer = left.length < right.length ? right : left;
|
|
11381
|
+
let shorterIndex = 0;
|
|
11382
|
+
let longerIndex = 0;
|
|
11383
|
+
let edits = 0;
|
|
11384
|
+
while (shorterIndex < shorter.length && longerIndex < longer.length) {
|
|
11385
|
+
if (shorter[shorterIndex] === longer[longerIndex]) {
|
|
11386
|
+
shorterIndex += 1;
|
|
11387
|
+
longerIndex += 1;
|
|
11388
|
+
} else {
|
|
11389
|
+
edits += 1;
|
|
11390
|
+
if (edits > 1) return false;
|
|
11391
|
+
longerIndex += 1;
|
|
11392
|
+
}
|
|
11393
|
+
}
|
|
11394
|
+
return true;
|
|
11395
|
+
}
|
|
11396
|
+
function nearDiscoveryNameMatch(query, name) {
|
|
11397
|
+
const queryParts = normalizeDiscoveryName(query).split(/\s+/).filter(Boolean);
|
|
11398
|
+
const nameParts = normalizeDiscoveryName(name).split(/\s+/).filter(Boolean);
|
|
11399
|
+
if (queryParts.length !== nameParts.length) return false;
|
|
11400
|
+
return queryParts.every((part, index) => {
|
|
11401
|
+
const target = nameParts[index];
|
|
11402
|
+
if (part === target) return true;
|
|
11403
|
+
if (part.length < 5 || target.length < 5) return false;
|
|
11404
|
+
return damerauLevenshteinAtMostOne(part, target);
|
|
11405
|
+
});
|
|
11406
|
+
}
|
|
11407
|
+
function lowerValues(values) {
|
|
11408
|
+
return values.map((value) => value.toLowerCase());
|
|
8921
11409
|
}
|
|
8922
|
-
function discoveryMatches(item, query) {
|
|
11410
|
+
function discoveryMatches(item, query, options = {}) {
|
|
8923
11411
|
if (!query) return [];
|
|
8924
11412
|
const matches = [];
|
|
8925
11413
|
const tokens = discoveryTokens(query);
|
|
8926
|
-
const
|
|
11414
|
+
const lower = tokens.length > 0 ? tokens.join(" ") : query.toLowerCase().trim();
|
|
11415
|
+
const rawLower = query.toLowerCase().trim();
|
|
8927
11416
|
const idParts = item.id.split(/[:/]/);
|
|
8928
11417
|
const names = [
|
|
8929
11418
|
item.id,
|
|
@@ -8949,33 +11438,66 @@ function discoveryMatches(item, query) {
|
|
|
8949
11438
|
...item.tags
|
|
8950
11439
|
].filter((value) => hasString(value));
|
|
8951
11440
|
const sourceValues = [item.source?.path, item.source?.repo_relative_path, item.source?.locator].filter((value) => hasString(value));
|
|
11441
|
+
const title = recordTitle(item);
|
|
11442
|
+
const summary = [item.search?.summary, recordText(item)].filter((value) => hasString(value)).join("\n");
|
|
11443
|
+
const type = item.search?.type ?? item.type;
|
|
11444
|
+
const tags = [...item.search?.tags ?? [], ...item.tags];
|
|
11445
|
+
const useMultiToken = tokens.length > 1;
|
|
11446
|
+
const minHits = useMultiToken ? options.relaxOverlap ? 1 : Math.ceil(tokens.length / 2) : 1;
|
|
11447
|
+
const overlapOK = (hits) => useMultiToken && hits >= minHits;
|
|
11448
|
+
const addInfo = (match) => addDiscoveryMatch(matches, { ...match, score: 0 });
|
|
11449
|
+
const addScore = (score2) => {
|
|
11450
|
+
if (score2 > 0) addDiscoveryMatch(matches, { field: "id", value: item.id, score: score2, reason: "aiwg discovery score" });
|
|
11451
|
+
};
|
|
8952
11452
|
for (const name of names) {
|
|
8953
|
-
|
|
8954
|
-
|
|
8955
|
-
|
|
8956
|
-
|
|
8957
|
-
}
|
|
8958
|
-
|
|
11453
|
+
if (normalizeDiscoveryName(query) === normalizeDiscoveryName(name)) {
|
|
11454
|
+
addInfo({ field: "id", value: name, reason: "exact canonical name" });
|
|
11455
|
+
addScore(1.001);
|
|
11456
|
+
return matches;
|
|
11457
|
+
}
|
|
11458
|
+
if (nearDiscoveryNameMatch(query, name)) {
|
|
11459
|
+
addInfo({ field: "id", value: name, reason: "near canonical name" });
|
|
11460
|
+
addScore(0.951);
|
|
11461
|
+
return matches;
|
|
8959
11462
|
}
|
|
8960
11463
|
}
|
|
8961
|
-
|
|
8962
|
-
const
|
|
8963
|
-
|
|
8964
|
-
|
|
8965
|
-
|
|
8966
|
-
|
|
11464
|
+
let score = 0;
|
|
11465
|
+
const scoreText = (field, value, reason, exactScore, overlapScore, weight, exactBonus = 0) => {
|
|
11466
|
+
const normalized = value.toLowerCase();
|
|
11467
|
+
if (normalized.includes(lower)) {
|
|
11468
|
+
score += exactScore * weight;
|
|
11469
|
+
if (normalized === lower) score += exactBonus;
|
|
11470
|
+
addInfo({ field, value, reason });
|
|
11471
|
+
} else if (useMultiToken) {
|
|
11472
|
+
const hits = tokens.filter((token) => normalized.includes(token)).length;
|
|
11473
|
+
if (overlapOK(hits)) {
|
|
11474
|
+
score += overlapScore * weight * (hits / tokens.length);
|
|
11475
|
+
addInfo({ field, value, reason });
|
|
11476
|
+
}
|
|
11477
|
+
}
|
|
11478
|
+
};
|
|
11479
|
+
for (const trigger of lowerValues(triggers)) {
|
|
11480
|
+
if (trigger === lower || trigger === rawLower) {
|
|
11481
|
+
addInfo({ field: "facet", value: trigger, reason: "trigger phrase" });
|
|
11482
|
+
addScore(1.0008);
|
|
11483
|
+
return matches;
|
|
11484
|
+
}
|
|
8967
11485
|
}
|
|
8968
|
-
for (const
|
|
8969
|
-
|
|
8970
|
-
if (overlap > 0) addDiscoveryMatch(matches, { field: "concept", value: capability, score: 22 * overlap, reason: "capability overlap" });
|
|
11486
|
+
for (const trigger of triggers) {
|
|
11487
|
+
scoreText("facet", trigger, "trigger phrase", 0.25, 0.06, 4);
|
|
8971
11488
|
}
|
|
8972
|
-
const
|
|
8973
|
-
|
|
8974
|
-
|
|
11489
|
+
for (const capability of capabilities) scoreText("concept", capability, "capability overlap", 0.2, 0.1, 2);
|
|
11490
|
+
scoreText("title", title, "title token overlap", 0.3, 0.08, 3, 0.2);
|
|
11491
|
+
for (const tag of tags) scoreText("tag", tag, "tag token overlap", 0.2, 0.05, 2);
|
|
11492
|
+
if (summary) scoreText("text", summary, "body token overlap", 0.15, 0.04, 1);
|
|
8975
11493
|
for (const source of sourceValues) {
|
|
8976
|
-
|
|
8977
|
-
|
|
11494
|
+
scoreText("source", source, "path overlap", 0.1, 0.03, 1);
|
|
11495
|
+
}
|
|
11496
|
+
if (type.toLowerCase().includes(lower)) {
|
|
11497
|
+
score += 0.1;
|
|
11498
|
+
addInfo({ field: "facet", value: type, reason: "type overlap" });
|
|
8978
11499
|
}
|
|
11500
|
+
addScore(Math.min(score, 1));
|
|
8979
11501
|
return matches;
|
|
8980
11502
|
}
|
|
8981
11503
|
function rankMatches(matches, weights) {
|
|
@@ -9001,13 +11523,13 @@ function createSnippet(item, matches, q, maxLength) {
|
|
|
9001
11523
|
const firstMatch = textMatch ?? titleMatch ?? matches[0];
|
|
9002
11524
|
return clipSnippet(firstMatch?.value ?? recordText(item), q, maxLength);
|
|
9003
11525
|
}
|
|
9004
|
-
function createRankedEntries(items, q, options, ordinalBase = 0) {
|
|
11526
|
+
function createRankedEntries(items, q, options, ordinalBase = 0, discoveryOptions = {}) {
|
|
9005
11527
|
const weights = { ...DEFAULT_QUERY_WEIGHTS, ...options.weights };
|
|
9006
11528
|
const profile = options.searchProfile ?? "default";
|
|
9007
11529
|
return items.map((item, ordinal) => ({
|
|
9008
11530
|
item,
|
|
9009
11531
|
ordinal: ordinalBase + ordinal,
|
|
9010
|
-
matches: profile === "aiwg-discovery" ? discoveryMatches(item, q) : queryMatches(item, q)
|
|
11532
|
+
matches: profile === "aiwg-discovery" ? discoveryMatches(item, q, discoveryOptions) : queryMatches(item, q)
|
|
9011
11533
|
})).filter(({ item, matches }) => {
|
|
9012
11534
|
if (q && matches.length === 0) return false;
|
|
9013
11535
|
if (options.types && !options.types.includes(item.type)) return false;
|
|
@@ -9057,8 +11579,7 @@ function queryAiwgFortemiIndex(index, query = "", options = {}) {
|
|
|
9057
11579
|
const q = query.trim().toLowerCase();
|
|
9058
11580
|
const entries = createRankedEntries(index.items, q, options);
|
|
9059
11581
|
if (entries.length === 0 && q && options.searchProfile === "aiwg-discovery") {
|
|
9060
|
-
|
|
9061
|
-
return createQueryResultFromRankedEntries(createRankedEntries(index.items, relaxed, options), relaxed, options);
|
|
11582
|
+
return createQueryResultFromRankedEntries(createRankedEntries(index.items, q, options, 0, { relaxOverlap: true }), q, options);
|
|
9062
11583
|
}
|
|
9063
11584
|
return createQueryResultFromRankedEntries(entries, q, options);
|
|
9064
11585
|
}
|
|
@@ -9079,7 +11600,8 @@ function cosineSimilarity2(left, right) {
|
|
|
9079
11600
|
}
|
|
9080
11601
|
function validateAiwgStaticEmbeddingSet(value) {
|
|
9081
11602
|
const errors = [];
|
|
9082
|
-
const data = value;
|
|
11603
|
+
const data = isPlainRecord(value) ? value : {};
|
|
11604
|
+
if (!isPlainRecord(value)) errors.push("embedding set must be an object");
|
|
9083
11605
|
if (data?.schema_version !== "aiwg.fortemi.embedding.set.v1") errors.push("schema_version must be aiwg.fortemi.embedding.set.v1");
|
|
9084
11606
|
if (!hasString(data?.id)) errors.push("id is required");
|
|
9085
11607
|
if (!hasString(data?.model)) errors.push("model is required");
|
|
@@ -9087,7 +11609,12 @@ function validateAiwgStaticEmbeddingSet(value) {
|
|
|
9087
11609
|
if (!hasString(data?.generated_at)) errors.push("generated_at is required");
|
|
9088
11610
|
if (!hasString(data?.granularity)) errors.push("granularity is required");
|
|
9089
11611
|
if (!Array.isArray(data?.embeddings)) errors.push("embeddings must be an array");
|
|
9090
|
-
|
|
11612
|
+
const embeddings = Array.isArray(data.embeddings) ? data.embeddings : [];
|
|
11613
|
+
for (const [index, embedding] of embeddings.entries()) {
|
|
11614
|
+
if (!isPlainRecord(embedding)) {
|
|
11615
|
+
errors.push("embeddings[" + index + "] must be an object");
|
|
11616
|
+
continue;
|
|
11617
|
+
}
|
|
9091
11618
|
if (!hasString(embedding.record_id)) errors.push("embeddings[" + index + "].record_id is required");
|
|
9092
11619
|
if (!hasString(embedding.input_hash)) errors.push("embeddings[" + index + "].input_hash is required");
|
|
9093
11620
|
if (!Array.isArray(embedding.embedding)) errors.push("embeddings[" + index + "].embedding must be an array");
|
|
@@ -9117,7 +11644,10 @@ function queryAiwgSemanticIndex(index, embeddingSet, queryEmbedding, options = {
|
|
|
9117
11644
|
}).filter((result) => result !== null && result.score >= (options.minScore ?? -1)).sort((left, right) => right.score - left.score || left.item.id.localeCompare(right.item.id)).slice(offset, offset + limit);
|
|
9118
11645
|
}
|
|
9119
11646
|
function queryAiwgHybridIndex(index, embeddingSet, query, queryEmbedding, options = {}) {
|
|
9120
|
-
const
|
|
11647
|
+
const lexicalOptions = { ...options };
|
|
11648
|
+
delete lexicalOptions.limit;
|
|
11649
|
+
delete lexicalOptions.offset;
|
|
11650
|
+
const lexical = queryAiwgFortemiIndex(index, query, { ...lexicalOptions, rank: true });
|
|
9121
11651
|
const semantic = queryAiwgSemanticIndex(index, embeddingSet, queryEmbedding, { limit: index.items.length });
|
|
9122
11652
|
const lexicalWeight = options.lexicalWeight ?? 0.5;
|
|
9123
11653
|
const semanticWeight = options.semanticWeight ?? 0.5;
|
|
@@ -9187,6 +11717,7 @@ function matchSetCacheKey(q, options) {
|
|
|
9187
11717
|
concepts: options.concepts ?? null,
|
|
9188
11718
|
privacy: options.privacy ?? null,
|
|
9189
11719
|
rel: options.relationshipTargetId ?? null,
|
|
11720
|
+
searchProfile: options.searchProfile ?? "default",
|
|
9190
11721
|
weights: { ...DEFAULT_QUERY_WEIGHTS, ...options.weights }
|
|
9191
11722
|
});
|
|
9192
11723
|
}
|
|
@@ -9275,15 +11806,21 @@ function edgeFromRelationship(sourceId, relationship) {
|
|
|
9275
11806
|
}
|
|
9276
11807
|
function relationshipMatches(edge, options = {}) {
|
|
9277
11808
|
const type = relationshipTypeFilter(options);
|
|
9278
|
-
const direction = options.direction ?? "both";
|
|
9279
11809
|
if (type && edge.type !== type) return false;
|
|
9280
11810
|
if (options.relationshipDirection && edge.direction !== options.relationshipDirection) return false;
|
|
9281
11811
|
if (options.sourceId && edge.source_id !== options.sourceId) return false;
|
|
9282
11812
|
if (options.targetId && edge.target_id !== options.targetId) return false;
|
|
9283
|
-
if (
|
|
9284
|
-
if (direction === "in" && options.sourceId && edge.source_id !== options.sourceId) return false;
|
|
11813
|
+
if (options.endpointId && edge.source_id !== options.endpointId && edge.target_id !== options.endpointId) return false;
|
|
9285
11814
|
return true;
|
|
9286
11815
|
}
|
|
11816
|
+
function assertRelationshipDirectionAnchor(options) {
|
|
11817
|
+
if (options.direction === "out" && !options.sourceId) {
|
|
11818
|
+
throw new Error("relationship direction 'out' requires sourceId");
|
|
11819
|
+
}
|
|
11820
|
+
if (options.direction === "in" && !options.targetId) {
|
|
11821
|
+
throw new Error("relationship direction 'in' requires targetId");
|
|
11822
|
+
}
|
|
11823
|
+
}
|
|
9287
11824
|
function nodeSummary(item) {
|
|
9288
11825
|
return { id: item.id, type: item.type, title: recordTitle(item) };
|
|
9289
11826
|
}
|
|
@@ -9291,6 +11828,7 @@ function addNode(nodes, item) {
|
|
|
9291
11828
|
if (item) nodes.set(item.id, nodeSummary(item));
|
|
9292
11829
|
}
|
|
9293
11830
|
function relationshipResultFromRecords(records, options = {}) {
|
|
11831
|
+
assertRelationshipDirectionAnchor(options);
|
|
9294
11832
|
const byId = new Map(records.map((record) => [record.id, record]));
|
|
9295
11833
|
const edges = [];
|
|
9296
11834
|
for (const record of records) {
|
|
@@ -9300,7 +11838,8 @@ function relationshipResultFromRecords(records, options = {}) {
|
|
|
9300
11838
|
edges.push(edge);
|
|
9301
11839
|
}
|
|
9302
11840
|
}
|
|
9303
|
-
const
|
|
11841
|
+
const sortedEdges = edges.sort((left, right) => left.source_id.localeCompare(right.source_id) || left.target_id.localeCompare(right.target_id) || left.type.localeCompare(right.type));
|
|
11842
|
+
const limitedEdges = options.limit ? sortedEdges.slice(0, options.limit) : sortedEdges;
|
|
9304
11843
|
const nodes = /* @__PURE__ */ new Map();
|
|
9305
11844
|
for (const edge of limitedEdges) {
|
|
9306
11845
|
addNode(nodes, byId.get(edge.source_id));
|
|
@@ -9317,7 +11856,8 @@ function neighborQueryOptions(id, options = {}) {
|
|
|
9317
11856
|
return {
|
|
9318
11857
|
...options,
|
|
9319
11858
|
...direction === "out" ? { sourceId: id } : {},
|
|
9320
|
-
...direction === "in" ? { targetId: id } : {}
|
|
11859
|
+
...direction === "in" ? { targetId: id } : {},
|
|
11860
|
+
...direction === "both" ? { endpointId: id } : {}
|
|
9321
11861
|
};
|
|
9322
11862
|
}
|
|
9323
11863
|
function filterNeighborResult(id, result, options = {}) {
|
|
@@ -9626,13 +12166,14 @@ function createAiwgIndexController(initialIndex) {
|
|
|
9626
12166
|
}
|
|
9627
12167
|
function aiwgFortemiIndexToCommunityGraph(index, options = {}) {
|
|
9628
12168
|
const ids = new Set(index.items.map((item) => item.id));
|
|
9629
|
-
const relationshipWeights = options.relationshipWeights ??
|
|
12169
|
+
const relationshipWeights = options.relationshipWeights ?? /* @__PURE__ */ Object.create(null);
|
|
9630
12170
|
const edgeCounts = /* @__PURE__ */ new Map();
|
|
9631
12171
|
for (const item of index.items) {
|
|
9632
12172
|
for (const relationship of item.relationships) {
|
|
9633
12173
|
if (!ids.has(relationship.target_id) && !options.includeDanglingRelationships) continue;
|
|
9634
12174
|
const kind = relationship.type;
|
|
9635
|
-
const
|
|
12175
|
+
const configuredWeight = Object.prototype.hasOwnProperty.call(relationshipWeights, kind) ? relationshipWeights[kind] : void 0;
|
|
12176
|
+
const baseWeight = typeof configuredWeight === "number" && Number.isFinite(configuredWeight) ? configuredWeight : 1;
|
|
9636
12177
|
const key = `${item.id}\0${relationship.target_id}\0${kind}`;
|
|
9637
12178
|
const existing = edgeCounts.get(key);
|
|
9638
12179
|
if (existing) existing.weight += baseWeight;
|
|
@@ -9669,8 +12210,8 @@ function communityIdsFor(item, options) {
|
|
|
9669
12210
|
}
|
|
9670
12211
|
|
|
9671
12212
|
// src/index.ts
|
|
9672
|
-
var VERSION = "2026.7.
|
|
12213
|
+
var VERSION = "2026.7.4";
|
|
9673
12214
|
|
|
9674
|
-
export { AIWG_SCAN_REQUIRED_FIELDS, ArchiveManager, AttachmentsRepository, CURRENT_MIGRATION_HEAD, CURRENT_SHARD_VERSION, CapabilityManager, CaptureKnowledgeInputSchema, CollectionsRepository, CommunitiesRepository, DB_SNAPSHOT_SCHEMA_VERSION, DbSnapshotVersionError, EMBED_REQUEST_KIND, EMBED_RESPONSE_KIND, EmbeddingSetsRepository, FallbackRouter, FortemiToolManifest, GetNoteInputSchema, GraphRepository, JOB_CAPABILITIES, JOB_PRIORITIES, JobQueueWorker, LOCAL_ENDPOINTS, LinksRepository, ListNotesInputSchema, ManageArchiveInputSchema, ManageAttachmentsInputSchema, ManageCapabilitiesInputSchema, ManageCollectionsInputSchema, ManageLinksInputSchema, ManageNoteInputSchema, ManageTagsInputSchema, MemoryBlobStore, MigrationRunner, NotesRepository, OpenAICompatibleProvider, PGliteStorageBackend, PGliteStorageBackendFactory, PGliteWorkerClient, PGliteWorkerStorageBackend, PGliteWorkerStorageBackendFactory, ProvenanceRepository, ProviderRegistry, SHARD_FORMAT, SUPPORTED_PGLITE_VERSION, SearchInputSchema, SearchRepository, SkosRepository, TagsRepository, TransactionProxy, TypedEventBus, VERSION, aiRevisionHandler, aiwgFortemiIndexToCommunityGraph, allMigrations, appendPluginScript, assertAiwgFortemiChunkManifest, assertAiwgFortemiChunkPart, assertAiwgFortemiIndexExport, assertAiwgStaticEmbeddingSet, buildAiwgChunkedIndex, buildAiwgStaticEmbeddingSet, buildNoteConditions, buildPluginCsp, captureKnowledge, chunkText, classifyError, classifyModel, clearPrefetchedShard, collectionFromShard, collectionToShard, computeHash, computeSri, conceptTaggingHandler, cosineSimilarity, createAiwgFetchChunkLoader, createAiwgFetchDetailLoader, createAiwgIndexController, createAiwgReviewDecisionExport, createBlobStore, createCosineSemanticProvider, createCspReportHandler, createFortemi, createLegacyProvider, createPGliteBackend, createPGliteInstance, createRemoteBackend, createRoutes, createShardBackend, createWorkerEmbedFunction, defaultStorageBackendFactory, detectCommunities, detectGpuCapabilities, detectInferenceCapabilities, discoverLocalProviders, dumpDbSnapshot, embeddingFromShard, embeddingGenerationHandler, embeddingSetFromShard, embeddingSetMemberToShard, embeddingSetToShard, embeddingToShard, enqueueFullWorkflow, enqueueJob, enqueueNoteCreationJobs, estimateModelFit, estimateVramMB, estimateVramTier, exportShard, fetchPluginScript, filterAiwgRecordsByPrivacy, findAiwgStaticDuplicatePairs, fortemiManifest, fromPrefetched, generateId, getAiwgFortemiFacets, getEmbedFunction, getFortemiBridge, getFortemiSecretStore, getJobQueueStatus, getLlmFunction, getNote, getPrefetchedSha256, handleEmbedRequests, hasFortemiSecureSecrets, importShard, isPluginScriptAllowed, isShardPrefetched, linkFromShard, linkToShard, linkingHandler, listNotes, manageArchive, manageAttachments, manageCapabilities, manageCollections, manageLinks, manageNote, manageTags, matchRoute, noteFromShard, noteSkosTagToShard, noteToShard, openShard, packTarGz, parseCspReport, prefetchShard, provenanceEdgeToShard, queryAiwgFortemiIndex, queryAiwgHybridIndex, queryAiwgSemanticIndex, registerLlmCapability, registerSemanticCapability, registerSemanticCapabilityWorker, registerServiceWorker, restoreDbSnapshot, searchTool, selectBackend, selectLlmModel, setEmbedFunction, setLlmFunction, sha256Hex, skosConceptToShard, skosRelationToShard, skosSchemeToShard, suggestTags,
|
|
12215
|
+
export { AIWG_SCAN_REQUIRED_FIELDS, ArchiveManager, AttachmentsRepository, CURRENT_MIGRATION_HEAD, CURRENT_SHARD_VERSION, CapabilityManager, CaptureKnowledgeInputSchema, CollectionsRepository, CommunitiesRepository, DB_SNAPSHOT_SCHEMA_VERSION, DbSnapshotVersionError, EMBED_REQUEST_KIND, EMBED_RESPONSE_KIND, EmbeddingSetsRepository, FORTEMI_COMPATIBILITY_PATH, FORTEMI_COMPATIBILITY_STATES, FORTEMI_REQUIRED_COMPATIBILITY_CAPABILITIES, FallbackRouter, FortemiToolManifest, GetNoteInputSchema, GraphRepository, JOB_CAPABILITIES, JOB_PRIORITIES, JobQueueWorker, LOCAL_ENDPOINTS, LinksRepository, ListNotesInputSchema, ManageArchiveInputSchema, ManageAttachmentsInputSchema, ManageCapabilitiesInputSchema, ManageCollectionsInputSchema, ManageLinksInputSchema, ManageNoteInputSchema, ManageTagsInputSchema, MemoryBlobStore, MigrationRunner, NotesRepository, OpenAICompatibleProvider, PGliteStorageBackend, PGliteStorageBackendFactory, PGliteWorkerClient, PGliteWorkerStorageBackend, PGliteWorkerStorageBackendFactory, ProvenanceRepository, ProviderRegistry, SHARD_FORMAT, SUPPORTED_PGLITE_VERSION, SearchInputSchema, SearchRepository, SkosRepository, TagsRepository, TransactionProxy, TypedEventBus, VERSION, aiRevisionHandler, aiwgFortemiIndexToCommunityGraph, allMigrations, appendPluginScript, assertAiwgFortemiChunkManifest, assertAiwgFortemiChunkPart, assertAiwgFortemiIndexExport, assertAiwgStaticEmbeddingSet, assertShardComponentRecord, buildAiwgChunkedIndex, buildAiwgStaticEmbeddingSet, buildNoteConditions, buildPluginCsp, captureKnowledge, chunkText, classifyError, classifyModel, clearPrefetchedShard, collectionFromShard, collectionToShard, computeBlobHash, computeHash, computeSri, conceptTaggingHandler, cosineSimilarity, createAiwgFetchChunkLoader, createAiwgFetchDetailLoader, createAiwgIndexController, createAiwgReviewDecisionExport, createBlobStore, createCosineSemanticProvider, createCspReportHandler, createFortemi, createLegacyProvider, createPGliteBackend, createPGliteInstance, createRemoteBackend, createRoutes, createShardBackend, createWorkerEmbedFunction, defaultStorageBackendFactory, detectCommunities, detectGpuCapabilities, detectInferenceCapabilities, discoverLocalProviders, dumpDbSnapshot, embeddingConfigToShard, embeddingFromShard, embeddingGenerationHandler, embeddingSetFromShard, embeddingSetMemberToShard, embeddingSetToShard, embeddingToShard, enqueueFullWorkflow, enqueueJob, enqueueNoteCreationJobs, estimateModelFit, estimateVramMB, estimateVramTier, exportShard, fetchAndValidateFortemiCompatibility, fetchPluginScript, filterAiwgRecordsByPrivacy, findAiwgStaticDuplicatePairs, formatFortemiCompatibilitySummary, fortemiCompatibilityUrl, fortemiManifest, fromPrefetched, generateId, getAiwgFortemiFacets, getAiwgFortemiIndexExportSchema, getEmbedFunction, getFortemiBridge, getFortemiSecretStore, getJobQueueStatus, getKnowledgeShardSchema, getLlmFunction, getNote, getPrefetchedSha256, handleEmbedRequests, hasFortemiSecureSecrets, importShard, isPluginScriptAllowed, isShardPrefetched, linkFromShard, linkToShard, linkingHandler, listNotes, manageArchive, manageAttachments, manageCapabilities, manageCollections, manageLinks, manageNote, manageTags, matchRoute, noteFromShard, noteSkosTagToShard, noteToShard, openShard, packTarGz, parseCspReport, prefetchShard, provenanceEdgeToShard, queryAiwgFortemiIndex, queryAiwgHybridIndex, queryAiwgSemanticIndex, registerLlmCapability, registerSemanticCapability, registerSemanticCapabilityWorker, registerServiceWorker, restoreDbSnapshot, searchTool, selectBackend, selectLlmModel, setEmbedFunction, setLlmFunction, sha256Hex, skosConceptToShard, skosRelationToShard, skosSchemeToShard, suggestTags, tagsToShard, templateToShard, titleGenerationHandler, unpackTarGz, unregisterLlmCapability, unregisterSemanticCapability, urlLinkToShard, validateAiwgFortemiChunkManifest, validateAiwgFortemiChunkPart, validateAiwgFortemiIndexExport, validateAiwgFortemiIndexExportSchema, validateAiwgFortemiProjectedRecordSchema, validateAiwgStaticEmbeddingSet, validateChecksums, validateFortemiCompatibilityResponse, validateShardArchive, validateShardComponentRecord, validateShardManifest, verifyDbSnapshotMeta, verifySri };
|
|
9675
12216
|
//# sourceMappingURL=index.js.map
|
|
9676
12217
|
//# sourceMappingURL=index.js.map
|