@fortemi/core 2026.7.13 → 2026.7.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +41 -2
- package/dist/aiwg-index-schema.d.ts +0 -1
- package/dist/aiwg-index-schema.js +49 -0
- package/dist/aiwg-index-schema.js.map +1 -1
- package/dist/{aiwg-index-shard-BhF8kKAU.d.ts → aiwg-index-shard-BCdaz0X1.d.ts} +26 -0
- package/dist/aiwg-index-shard.d.ts +1 -1
- package/dist/aiwg-index-shard.js +863 -28
- package/dist/aiwg-index-shard.js.map +1 -1
- package/dist/aiwg-index.d.ts +53 -28
- package/dist/aiwg-index.js +8 -8
- package/dist/aiwg-index.js.map +1 -1
- package/dist/index.d.ts +548 -577
- package/dist/index.js +1600 -1245
- package/dist/index.js.map +1 -1
- package/package.json +9 -3
package/dist/index.js
CHANGED
|
@@ -3493,6 +3493,7 @@ function createFortemi(config) {
|
|
|
3493
3493
|
|
|
3494
3494
|
// src/server-compatibility.ts
|
|
3495
3495
|
var FORTEMI_COMPATIBILITY_PATH = "/api/v1/system/compatibility";
|
|
3496
|
+
var FORTEMI_SERVER_COMPATIBILITY_REVISION = "2026-07-06";
|
|
3496
3497
|
var FORTEMI_COMPATIBILITY_STATES = [
|
|
3497
3498
|
"available",
|
|
3498
3499
|
"degraded",
|
|
@@ -7634,6 +7635,164 @@ var knowledge_shard_v2_implementation_receipt_default = {
|
|
|
7634
7635
|
schemaBundleSha256: "66dee80876c73fdc8756541c72e96ae189c098113a831c849d619381c4121c02"
|
|
7635
7636
|
}};
|
|
7636
7637
|
|
|
7638
|
+
// schemas/knowledge-shard-v2.cross-repository.receipt.json
|
|
7639
|
+
var knowledge_shard_v2_cross_repository_receipt_default = {
|
|
7640
|
+
cells: [
|
|
7641
|
+
{
|
|
7642
|
+
id: "pglite-full-v1-to-pglite",
|
|
7643
|
+
producer: "react-pglite",
|
|
7644
|
+
consumer: "react-pglite",
|
|
7645
|
+
status: "passed",
|
|
7646
|
+
tests: [
|
|
7647
|
+
"full-v1-store: validates, persists all components and bytes, and converges on re-export",
|
|
7648
|
+
"full-v1-store: produces and converges a complete full-v1 archive from live PGlite state",
|
|
7649
|
+
"full-v1-store: rejects corrupt, incomplete, oversized, and transactional-failure inputs"
|
|
7650
|
+
],
|
|
7651
|
+
coverage: [
|
|
7652
|
+
"all-33-components",
|
|
7653
|
+
"all-34-count-fields",
|
|
7654
|
+
"attachment-bytes",
|
|
7655
|
+
"signatures",
|
|
7656
|
+
"identities",
|
|
7657
|
+
"relationships",
|
|
7658
|
+
"timestamps",
|
|
7659
|
+
"tombstones",
|
|
7660
|
+
"absent-null-empty-value",
|
|
7661
|
+
"768-dimensional-embeddings",
|
|
7662
|
+
"embedding-contract-lineage",
|
|
7663
|
+
"skos",
|
|
7664
|
+
"provenance",
|
|
7665
|
+
"graph-community",
|
|
7666
|
+
"current-minus-two",
|
|
7667
|
+
"current",
|
|
7668
|
+
"next-major-rejection",
|
|
7669
|
+
"malformed-input",
|
|
7670
|
+
"tampered-input",
|
|
7671
|
+
"resource-limits",
|
|
7672
|
+
"repeated-imports",
|
|
7673
|
+
"zero-mutation-on-rejection"
|
|
7674
|
+
]
|
|
7675
|
+
},
|
|
7676
|
+
{
|
|
7677
|
+
id: "aiwg-full-v1-to-pglite",
|
|
7678
|
+
producer: "aiwg-index-converter",
|
|
7679
|
+
consumer: "react-pglite",
|
|
7680
|
+
status: "passed",
|
|
7681
|
+
tests: [
|
|
7682
|
+
"aiwg-index-full-shard: verifies the committed source, implementation, archive, and authority receipt",
|
|
7683
|
+
"aiwg-index-full-shard: imports through PGlite and converges on exact logical files",
|
|
7684
|
+
"portable-contract: validates rejection, presence, signature, limit, and mutation invariants"
|
|
7685
|
+
],
|
|
7686
|
+
coverage: [
|
|
7687
|
+
"all-33-components",
|
|
7688
|
+
"all-34-count-fields",
|
|
7689
|
+
"attachment-bytes",
|
|
7690
|
+
"signatures",
|
|
7691
|
+
"identities",
|
|
7692
|
+
"relationships",
|
|
7693
|
+
"timestamps",
|
|
7694
|
+
"tombstones",
|
|
7695
|
+
"absent-null-empty-value",
|
|
7696
|
+
"768-dimensional-embeddings",
|
|
7697
|
+
"embedding-contract-lineage",
|
|
7698
|
+
"skos",
|
|
7699
|
+
"provenance",
|
|
7700
|
+
"graph-community",
|
|
7701
|
+
"current-minus-two",
|
|
7702
|
+
"current",
|
|
7703
|
+
"next-major-rejection",
|
|
7704
|
+
"malformed-input",
|
|
7705
|
+
"tampered-input",
|
|
7706
|
+
"resource-limits",
|
|
7707
|
+
"repeated-imports",
|
|
7708
|
+
"zero-mutation-on-rejection"
|
|
7709
|
+
]
|
|
7710
|
+
},
|
|
7711
|
+
{
|
|
7712
|
+
id: "pglite-full-v1-to-fortemi",
|
|
7713
|
+
producer: "react-pglite",
|
|
7714
|
+
consumer: "fortemi-postgresql",
|
|
7715
|
+
status: "passed",
|
|
7716
|
+
tests: [
|
|
7717
|
+
"tests::shard_schema_2_accepts_independent_react_and_aiwg_full_v1_archives",
|
|
7718
|
+
"tests::shard_schema_2_clean_destinations_preserve_independent_full_v1_archives",
|
|
7719
|
+
"tests::shard_schema_2_dispatch_covers_the_complete_authority_field_inventory"
|
|
7720
|
+
],
|
|
7721
|
+
coverage: [
|
|
7722
|
+
"all-33-components",
|
|
7723
|
+
"all-34-count-fields",
|
|
7724
|
+
"attachment-bytes",
|
|
7725
|
+
"signatures",
|
|
7726
|
+
"identities",
|
|
7727
|
+
"relationships",
|
|
7728
|
+
"timestamps",
|
|
7729
|
+
"tombstones",
|
|
7730
|
+
"absent-null-empty-value",
|
|
7731
|
+
"768-dimensional-embeddings",
|
|
7732
|
+
"embedding-contract-lineage",
|
|
7733
|
+
"skos",
|
|
7734
|
+
"provenance",
|
|
7735
|
+
"graph-community",
|
|
7736
|
+
"current-minus-two",
|
|
7737
|
+
"current",
|
|
7738
|
+
"next-major-rejection",
|
|
7739
|
+
"malformed-input",
|
|
7740
|
+
"tampered-input",
|
|
7741
|
+
"resource-limits",
|
|
7742
|
+
"repeated-imports",
|
|
7743
|
+
"zero-mutation-on-rejection"
|
|
7744
|
+
]
|
|
7745
|
+
},
|
|
7746
|
+
{
|
|
7747
|
+
id: "aiwg-full-v1-to-fortemi",
|
|
7748
|
+
producer: "aiwg-index-converter",
|
|
7749
|
+
consumer: "fortemi-postgresql",
|
|
7750
|
+
status: "passed",
|
|
7751
|
+
tests: [
|
|
7752
|
+
"tests::shard_schema_2_accepts_independent_react_and_aiwg_full_v1_archives",
|
|
7753
|
+
"tests::shard_schema_2_clean_destinations_preserve_independent_full_v1_archives",
|
|
7754
|
+
"tests::shard_schema_2_dispatch_covers_the_complete_authority_field_inventory"
|
|
7755
|
+
],
|
|
7756
|
+
coverage: [
|
|
7757
|
+
"all-33-components",
|
|
7758
|
+
"all-34-count-fields",
|
|
7759
|
+
"attachment-bytes",
|
|
7760
|
+
"signatures",
|
|
7761
|
+
"identities",
|
|
7762
|
+
"relationships",
|
|
7763
|
+
"timestamps",
|
|
7764
|
+
"tombstones",
|
|
7765
|
+
"absent-null-empty-value",
|
|
7766
|
+
"768-dimensional-embeddings",
|
|
7767
|
+
"embedding-contract-lineage",
|
|
7768
|
+
"skos",
|
|
7769
|
+
"provenance",
|
|
7770
|
+
"graph-community",
|
|
7771
|
+
"current-minus-two",
|
|
7772
|
+
"current",
|
|
7773
|
+
"next-major-rejection",
|
|
7774
|
+
"malformed-input",
|
|
7775
|
+
"tampered-input",
|
|
7776
|
+
"resource-limits",
|
|
7777
|
+
"repeated-imports",
|
|
7778
|
+
"zero-mutation-on-rejection"
|
|
7779
|
+
]
|
|
7780
|
+
}
|
|
7781
|
+
],
|
|
7782
|
+
advertisement: {
|
|
7783
|
+
backend: "pglite",
|
|
7784
|
+
schemaVersion: "2.0.0",
|
|
7785
|
+
profile: "full-v1",
|
|
7786
|
+
operations: [
|
|
7787
|
+
"import",
|
|
7788
|
+
"export"
|
|
7789
|
+
],
|
|
7790
|
+
enabled: true
|
|
7791
|
+
},
|
|
7792
|
+
claims: {
|
|
7793
|
+
suiteWide: false,
|
|
7794
|
+
recordStoreFullV1: false}};
|
|
7795
|
+
|
|
7637
7796
|
// src/shard/profile-registry.ts
|
|
7638
7797
|
var CORE_V1_COMPONENTS = [
|
|
7639
7798
|
"notes",
|
|
@@ -7643,7 +7802,38 @@ var CORE_V1_COMPONENTS = [
|
|
|
7643
7802
|
"links"
|
|
7644
7803
|
];
|
|
7645
7804
|
var ALL_PROFILES = ["core-v1", "full-v1", "record-v1"];
|
|
7805
|
+
var FULL_V1_RECEIPT_CELLS = [
|
|
7806
|
+
"pglite-full-v1-to-pglite",
|
|
7807
|
+
"aiwg-full-v1-to-pglite",
|
|
7808
|
+
"pglite-full-v1-to-fortemi",
|
|
7809
|
+
"aiwg-full-v1-to-fortemi"
|
|
7810
|
+
];
|
|
7811
|
+
var FULL_V1_RECEIPT_COVERAGE = [
|
|
7812
|
+
"all-33-components",
|
|
7813
|
+
"all-34-count-fields",
|
|
7814
|
+
"attachment-bytes",
|
|
7815
|
+
"signatures",
|
|
7816
|
+
"identities",
|
|
7817
|
+
"relationships",
|
|
7818
|
+
"timestamps",
|
|
7819
|
+
"tombstones",
|
|
7820
|
+
"absent-null-empty-value",
|
|
7821
|
+
"768-dimensional-embeddings",
|
|
7822
|
+
"embedding-contract-lineage",
|
|
7823
|
+
"skos",
|
|
7824
|
+
"provenance",
|
|
7825
|
+
"graph-community",
|
|
7826
|
+
"current-minus-two",
|
|
7827
|
+
"current",
|
|
7828
|
+
"next-major-rejection",
|
|
7829
|
+
"malformed-input",
|
|
7830
|
+
"tampered-input",
|
|
7831
|
+
"resource-limits",
|
|
7832
|
+
"repeated-imports",
|
|
7833
|
+
"zero-mutation-on-rejection"
|
|
7834
|
+
];
|
|
7646
7835
|
var FULL_V1_IMPLEMENTED = knowledge_shard_v2_implementation_receipt_default.authority.schemaBundleSha256 === knowledge_shard_v2_schema_receipt_default.schemaBundle.sha256;
|
|
7836
|
+
var FULL_V1_ADVERTISEMENT_RECEIPT_PASSED = knowledge_shard_v2_cross_repository_receipt_default.cells.length === FULL_V1_RECEIPT_CELLS.length && knowledge_shard_v2_cross_repository_receipt_default.cells.every((cell, index) => cell.id === FULL_V1_RECEIPT_CELLS[index] && cell.status === "passed" && JSON.stringify(cell.coverage) === JSON.stringify(FULL_V1_RECEIPT_COVERAGE)) && knowledge_shard_v2_cross_repository_receipt_default.advertisement.backend === "pglite" && knowledge_shard_v2_cross_repository_receipt_default.advertisement.schemaVersion === "2.0.0" && knowledge_shard_v2_cross_repository_receipt_default.advertisement.profile === "full-v1" && JSON.stringify(knowledge_shard_v2_cross_repository_receipt_default.advertisement.operations) === JSON.stringify(["import", "export"]) && knowledge_shard_v2_cross_repository_receipt_default.advertisement.enabled === true && knowledge_shard_v2_cross_repository_receipt_default.claims.suiteWide === false && knowledge_shard_v2_cross_repository_receipt_default.claims.recordStoreFullV1 === false;
|
|
7647
7837
|
var ALL_COMPONENTS = [
|
|
7648
7838
|
"notes",
|
|
7649
7839
|
"collections",
|
|
@@ -7736,12 +7926,14 @@ function getKnowledgeShardProfileRegistry() {
|
|
|
7736
7926
|
}
|
|
7737
7927
|
function createShardCapabilityReport(input) {
|
|
7738
7928
|
const requestedSchemaVersion = input.requestedSchemaVersion ?? null;
|
|
7739
|
-
const authorityPending = requestedSchemaVersion === "2.0.0";
|
|
7740
|
-
const advertisedProfiles = authorityPending ? [] : [...BACKEND_PROFILES[input.backend][input.operation]];
|
|
7741
7929
|
const requestedProfile = input.requestedProfile;
|
|
7742
7930
|
const knownProfile = requestedProfile && isKnownProfile(requestedProfile) ? requestedProfile : null;
|
|
7743
|
-
const
|
|
7744
|
-
const
|
|
7931
|
+
const schema2Requested = requestedSchemaVersion === "2.0.0";
|
|
7932
|
+
const receiptBackedSchema2FullV1 = schema2Requested && requestedProfile === "full-v1" && input.backend === "pglite" && FULL_V1_ADVERTISEMENT_RECEIPT_PASSED;
|
|
7933
|
+
const authorityPending = schema2Requested && !receiptBackedSchema2FullV1;
|
|
7934
|
+
const advertisedProfiles = schema2Requested ? receiptBackedSchema2FullV1 ? ["full-v1"] : [] : [...BACKEND_PROFILES[input.backend][input.operation]];
|
|
7935
|
+
const authorityStatus = requestedProfile === null ? "unprofiled" : knownProfile ? receiptBackedSchema2FullV1 ? "supported" : authorityPending ? "candidate" : registryEntry(knownProfile).authority_status : "unknown";
|
|
7936
|
+
const backendSupported = knownProfile !== null && (receiptBackedSchema2FullV1 || !authorityPending && advertisedProfiles.includes(knownProfile));
|
|
7745
7937
|
const backendComponents = new Set(BACKEND_COMPONENTS[input.backend][input.operation]);
|
|
7746
7938
|
const profileComponents = knownProfile === "core-v1" ? CORE_V1_COMPONENTS : BACKEND_COMPONENTS[input.backend][input.operation];
|
|
7747
7939
|
const supportedComponents = profileComponents.filter(
|
|
@@ -31143,6 +31335,86 @@ function validateOptionalRichMetadata(item, index, errors) {
|
|
|
31143
31335
|
if (item.compatibility !== void 0 && !isPlainRecord(item.compatibility)) {
|
|
31144
31336
|
errors.push("items[" + index + "].compatibility must be an object");
|
|
31145
31337
|
}
|
|
31338
|
+
if (item.operational_state !== void 0) {
|
|
31339
|
+
const at = "items[" + index + "].operational_state";
|
|
31340
|
+
if (!isPlainRecord(item.operational_state)) {
|
|
31341
|
+
errors.push(at + " must be an object");
|
|
31342
|
+
} else {
|
|
31343
|
+
const state = item.operational_state;
|
|
31344
|
+
const allowedClassifications = /* @__PURE__ */ new Set([
|
|
31345
|
+
"fresh",
|
|
31346
|
+
"historical",
|
|
31347
|
+
"superseded",
|
|
31348
|
+
"contradicted",
|
|
31349
|
+
"needs-source"
|
|
31350
|
+
]);
|
|
31351
|
+
const allowedConfidence = /* @__PURE__ */ new Set(["source", "candidate", "reviewed", "rejected"]);
|
|
31352
|
+
const stringFields = [
|
|
31353
|
+
"source_repo",
|
|
31354
|
+
"source_kind",
|
|
31355
|
+
"source_id",
|
|
31356
|
+
"observed_state",
|
|
31357
|
+
"observed_at",
|
|
31358
|
+
"source_updated_at",
|
|
31359
|
+
"evidence_url",
|
|
31360
|
+
"evidence_path",
|
|
31361
|
+
"observer",
|
|
31362
|
+
"stale_after"
|
|
31363
|
+
];
|
|
31364
|
+
const arrayFields = ["supersedes", "contradicts"];
|
|
31365
|
+
const allowedFields = /* @__PURE__ */ new Set([
|
|
31366
|
+
...stringFields,
|
|
31367
|
+
...arrayFields,
|
|
31368
|
+
"classification",
|
|
31369
|
+
"confidence",
|
|
31370
|
+
"current_action_selector"
|
|
31371
|
+
]);
|
|
31372
|
+
if (!allowedClassifications.has(String(state.classification))) {
|
|
31373
|
+
errors.push(at + ".classification must be fresh, historical, superseded, contradicted, or needs-source");
|
|
31374
|
+
}
|
|
31375
|
+
for (const field of stringFields) {
|
|
31376
|
+
if (state[field] !== void 0 && !hasString(state[field])) {
|
|
31377
|
+
errors.push(at + "." + field + " must be a non-empty string");
|
|
31378
|
+
}
|
|
31379
|
+
}
|
|
31380
|
+
for (const field of ["observed_at", "source_updated_at", "stale_after"]) {
|
|
31381
|
+
if (state[field] !== void 0 && (typeof state[field] !== "string" || Number.isNaN(Date.parse(state[field])))) {
|
|
31382
|
+
errors.push(at + "." + field + " must be a date-time string");
|
|
31383
|
+
}
|
|
31384
|
+
}
|
|
31385
|
+
for (const field of arrayFields) {
|
|
31386
|
+
if (!isOptionalStringArray(state[field])) {
|
|
31387
|
+
errors.push(at + "." + field + " must be a string array");
|
|
31388
|
+
}
|
|
31389
|
+
}
|
|
31390
|
+
if (state.confidence !== void 0 && !allowedConfidence.has(String(state.confidence))) {
|
|
31391
|
+
errors.push(at + ".confidence must be source, candidate, reviewed, or rejected");
|
|
31392
|
+
}
|
|
31393
|
+
if (state.current_action_selector !== void 0 && typeof state.current_action_selector !== "boolean") {
|
|
31394
|
+
errors.push(at + ".current_action_selector must be a boolean");
|
|
31395
|
+
}
|
|
31396
|
+
if (Object.keys(state).some((field) => !allowedFields.has(field))) {
|
|
31397
|
+
errors.push(at + " contains unsupported fields");
|
|
31398
|
+
}
|
|
31399
|
+
}
|
|
31400
|
+
}
|
|
31401
|
+
if (item.state_transfer !== void 0) {
|
|
31402
|
+
const at = "items[" + index + "].state_transfer";
|
|
31403
|
+
if (!isPlainRecord(item.state_transfer)) {
|
|
31404
|
+
errors.push(at + " must be an object");
|
|
31405
|
+
} else {
|
|
31406
|
+
const stateTransfer = item.state_transfer;
|
|
31407
|
+
const deletedAt = stateTransfer.deleted_at;
|
|
31408
|
+
if (!Object.hasOwn(stateTransfer, "deleted_at")) {
|
|
31409
|
+
errors.push(at + ".deleted_at is required");
|
|
31410
|
+
} else if (deletedAt !== null && (typeof deletedAt !== "string" || Number.isNaN(Date.parse(deletedAt)))) {
|
|
31411
|
+
errors.push(at + ".deleted_at must be a date-time string or null");
|
|
31412
|
+
}
|
|
31413
|
+
if (Object.keys(stateTransfer).some((field) => field !== "deleted_at")) {
|
|
31414
|
+
errors.push(at + " contains unsupported fields");
|
|
31415
|
+
}
|
|
31416
|
+
}
|
|
31417
|
+
}
|
|
31146
31418
|
}
|
|
31147
31419
|
function isPrivacyClassification(value) {
|
|
31148
31420
|
return value === "private" || value === "sanitized" || value === "public";
|
|
@@ -31172,7 +31444,9 @@ var V2_ONLY_RECORD_FIELDS = [
|
|
|
31172
31444
|
"skos_concepts",
|
|
31173
31445
|
"skos_relations",
|
|
31174
31446
|
"provenance_events",
|
|
31175
|
-
"compatibility"
|
|
31447
|
+
"compatibility",
|
|
31448
|
+
"operational_state",
|
|
31449
|
+
"state_transfer"
|
|
31176
31450
|
];
|
|
31177
31451
|
var V2_ONLY_SOURCE_FIELDS = ["origin", "generated", "checksum", "updated_at"];
|
|
31178
31452
|
var V2_ONLY_RELATIONSHIP_FIELDS = ["target_path", "direction", "metadata"];
|
|
@@ -32762,1178 +33036,233 @@ function communityIdsFor(item, options) {
|
|
|
32762
33036
|
if (item.concepts.length > 0) return item.concepts.map((concept) => `concept:${concept}`);
|
|
32763
33037
|
return [`type:${item.type}`];
|
|
32764
33038
|
}
|
|
32765
|
-
|
|
32766
|
-
|
|
32767
|
-
|
|
32768
|
-
|
|
32769
|
-
|
|
32770
|
-
|
|
32771
|
-
|
|
32772
|
-
|
|
32773
|
-
|
|
32774
|
-
|
|
32775
|
-
|
|
32776
|
-
|
|
32777
|
-
|
|
32778
|
-
|
|
32779
|
-
|
|
32780
|
-
|
|
32781
|
-
|
|
32782
|
-
|
|
32783
|
-
|
|
32784
|
-
|
|
32785
|
-
|
|
32786
|
-
|
|
32787
|
-
|
|
32788
|
-
|
|
32789
|
-
|
|
32790
|
-
|
|
32791
|
-
|
|
32792
|
-
|
|
32793
|
-
|
|
32794
|
-
|
|
32795
|
-
|
|
32796
|
-
|
|
32797
|
-
|
|
32798
|
-
|
|
32799
|
-
|
|
32800
|
-
|
|
32801
|
-
"
|
|
32802
|
-
|
|
32803
|
-
|
|
32804
|
-
|
|
32805
|
-
|
|
32806
|
-
|
|
32807
|
-
|
|
32808
|
-
|
|
32809
|
-
|
|
33039
|
+
|
|
33040
|
+
// schemas/aiwg-fortemi-index-export.schema.json
|
|
33041
|
+
var aiwg_fortemi_index_export_schema_default = {
|
|
33042
|
+
$schema: "https://json-schema.org/draft/2020-12/schema",
|
|
33043
|
+
$id: "https://aiwg.io/schemas/aiwg-fortemi-index-export.json",
|
|
33044
|
+
title: "AIWG Fortemi Index Export",
|
|
33045
|
+
type: "object",
|
|
33046
|
+
additionalProperties: false,
|
|
33047
|
+
required: ["schema_version", "generated_at", "source", "items"],
|
|
33048
|
+
properties: {
|
|
33049
|
+
schema_version: {
|
|
33050
|
+
enum: ["aiwg.fortemi.index.export.v1", "aiwg.fortemi.index.export.v2"]
|
|
33051
|
+
},
|
|
33052
|
+
generated_at: {
|
|
33053
|
+
type: "string",
|
|
33054
|
+
format: "date-time"
|
|
33055
|
+
},
|
|
33056
|
+
source: {
|
|
33057
|
+
type: "object",
|
|
33058
|
+
additionalProperties: false,
|
|
33059
|
+
required: ["repo", "privacy"],
|
|
33060
|
+
properties: {
|
|
33061
|
+
repo: {
|
|
33062
|
+
type: "string",
|
|
33063
|
+
minLength: 1
|
|
33064
|
+
},
|
|
33065
|
+
privacy: {
|
|
33066
|
+
$ref: "#/$defs/privacy"
|
|
33067
|
+
},
|
|
33068
|
+
graph: {
|
|
33069
|
+
type: "string",
|
|
33070
|
+
minLength: 1
|
|
33071
|
+
}
|
|
33072
|
+
}
|
|
33073
|
+
},
|
|
33074
|
+
compatibility: {
|
|
33075
|
+
type: "object",
|
|
33076
|
+
additionalProperties: false,
|
|
33077
|
+
required: ["previous_schema_version", "strategy"],
|
|
33078
|
+
properties: {
|
|
33079
|
+
previous_schema_version: {
|
|
33080
|
+
const: "aiwg.fortemi.index.export.v1"
|
|
33081
|
+
},
|
|
33082
|
+
strategy: {
|
|
33083
|
+
const: "supported"
|
|
33084
|
+
}
|
|
33085
|
+
}
|
|
33086
|
+
},
|
|
33087
|
+
items: {
|
|
33088
|
+
type: "array",
|
|
33089
|
+
items: {
|
|
33090
|
+
$ref: "#/$defs/record"
|
|
32810
33091
|
}
|
|
32811
|
-
);
|
|
32812
|
-
return record.search?.body ?? "";
|
|
32813
|
-
}
|
|
32814
|
-
if (record.chunks?.length) {
|
|
32815
|
-
addLoss(losses, "aiwg-content-chunk-projection", "Content was assembled from AIWG chunks.", {
|
|
32816
|
-
component: "notes",
|
|
32817
|
-
record_id: record.id,
|
|
32818
|
-
field_path: "/original_content",
|
|
32819
|
-
source_state: "absent",
|
|
32820
|
-
destination_capability: "required-string",
|
|
32821
|
-
action: "degrade"
|
|
32822
|
-
});
|
|
32823
|
-
return record.chunks.map((chunk) => {
|
|
32824
|
-
if (own(chunk, "text")) return chunk.text ?? "";
|
|
32825
|
-
if (own(chunk, "body")) return chunk.body ?? "";
|
|
32826
|
-
if (own(chunk, "summary")) return chunk.summary ?? "";
|
|
32827
|
-
return "";
|
|
32828
|
-
}).join("\n\n");
|
|
32829
|
-
}
|
|
32830
|
-
addLoss(
|
|
32831
|
-
losses,
|
|
32832
|
-
"aiwg-content-unavailable",
|
|
32833
|
-
"AIWG supplied no body; the required destination string is empty.",
|
|
32834
|
-
{
|
|
32835
|
-
component: "notes",
|
|
32836
|
-
record_id: record.id,
|
|
32837
|
-
field_path: "/original_content",
|
|
32838
|
-
source_state: "absent",
|
|
32839
|
-
destination_capability: "required-string",
|
|
32840
|
-
action: "default"
|
|
32841
33092
|
}
|
|
32842
|
-
|
|
32843
|
-
|
|
32844
|
-
}
|
|
32845
|
-
function createdAt(record, losses) {
|
|
32846
|
-
const source = timestamp(record.source.updated_at);
|
|
32847
|
-
if (source) return source;
|
|
32848
|
-
addLoss(
|
|
32849
|
-
losses,
|
|
32850
|
-
"aiwg-created-at-unavailable",
|
|
32851
|
-
"AIWG has no distinct creation timestamp; updated_at was used.",
|
|
33093
|
+
},
|
|
33094
|
+
allOf: [
|
|
32852
33095
|
{
|
|
32853
|
-
|
|
32854
|
-
|
|
32855
|
-
|
|
32856
|
-
|
|
32857
|
-
|
|
32858
|
-
action: "default"
|
|
32859
|
-
}
|
|
32860
|
-
);
|
|
32861
|
-
return new Date(record.updated_at).toISOString();
|
|
32862
|
-
}
|
|
32863
|
-
function safeUri(value) {
|
|
32864
|
-
if (!value) return null;
|
|
32865
|
-
try {
|
|
32866
|
-
return new URL(value).toString();
|
|
32867
|
-
} catch {
|
|
32868
|
-
return null;
|
|
32869
|
-
}
|
|
32870
|
-
}
|
|
32871
|
-
function schemeName(concept) {
|
|
32872
|
-
return concept.scheme && concept.scheme.length > 0 ? concept.scheme : "aiwg";
|
|
32873
|
-
}
|
|
32874
|
-
async function convertAiwgIndexToFullV1(index, options = {}) {
|
|
32875
|
-
const losses = [];
|
|
32876
|
-
const exportedAt = timestamp(options.createdAt) ?? new Date(index.generated_at).toISOString();
|
|
32877
|
-
const records = [...index.items].sort((a, b) => a.id.localeCompare(b.id));
|
|
32878
|
-
const noteIds = new Map(records.map((record) => [record.id, uuid("record", record.id)]));
|
|
32879
|
-
const rows = /* @__PURE__ */ new Map();
|
|
32880
|
-
for (const component of Object.keys(FULL_V1_COMPONENT_FILES)) rows.set(component, []);
|
|
32881
|
-
const tags = /* @__PURE__ */ new Set();
|
|
32882
|
-
for (const record of records) {
|
|
32883
|
-
const noteId = noteIds.get(record.id);
|
|
32884
|
-
const content = noteContent(record, losses);
|
|
32885
|
-
const sourceCreatedAt = createdAt(record, losses);
|
|
32886
|
-
const updatedAt = new Date(record.updated_at).toISOString();
|
|
32887
|
-
const recordTags = [...new Set(record.tags)].sort();
|
|
32888
|
-
recordTags.forEach((tag) => tags.add(tag));
|
|
32889
|
-
if (record.binary_sources?.length) {
|
|
32890
|
-
addLoss(
|
|
32891
|
-
losses,
|
|
32892
|
-
"aiwg-attachment-bytes-unavailable",
|
|
32893
|
-
"Attachment references have no bytes, so full-v1 attachment projections were omitted.",
|
|
32894
|
-
{
|
|
32895
|
-
component: "notes",
|
|
32896
|
-
count: record.binary_sources.length,
|
|
32897
|
-
record_id: record.id,
|
|
32898
|
-
field_path: "/attachments",
|
|
32899
|
-
source_state: "value",
|
|
32900
|
-
destination_capability: "mandatory-blob-sidecar",
|
|
32901
|
-
action: "omit"
|
|
33096
|
+
if: {
|
|
33097
|
+
properties: {
|
|
33098
|
+
schema_version: {
|
|
33099
|
+
const: "aiwg.fortemi.index.export.v1"
|
|
33100
|
+
}
|
|
32902
33101
|
}
|
|
32903
|
-
|
|
32904
|
-
|
|
32905
|
-
|
|
32906
|
-
|
|
32907
|
-
|
|
32908
|
-
|
|
32909
|
-
|
|
32910
|
-
|
|
32911
|
-
|
|
32912
|
-
|
|
32913
|
-
|
|
32914
|
-
|
|
32915
|
-
|
|
32916
|
-
|
|
32917
|
-
|
|
32918
|
-
|
|
32919
|
-
|
|
32920
|
-
|
|
32921
|
-
|
|
32922
|
-
|
|
33102
|
+
},
|
|
33103
|
+
then: {
|
|
33104
|
+
not: {
|
|
33105
|
+
required: ["compatibility"]
|
|
33106
|
+
},
|
|
33107
|
+
properties: {
|
|
33108
|
+
source: {
|
|
33109
|
+
not: {
|
|
33110
|
+
required: ["graph"]
|
|
33111
|
+
}
|
|
33112
|
+
},
|
|
33113
|
+
items: {
|
|
33114
|
+
items: {
|
|
33115
|
+
allOf: [
|
|
33116
|
+
{
|
|
33117
|
+
properties: {
|
|
33118
|
+
schema_version: {
|
|
33119
|
+
const: "aiwg.fortemi.index.record.v1"
|
|
33120
|
+
}
|
|
33121
|
+
}
|
|
33122
|
+
},
|
|
33123
|
+
{
|
|
33124
|
+
$ref: "#/$defs/v1RecordCompatibility"
|
|
33125
|
+
}
|
|
33126
|
+
]
|
|
33127
|
+
}
|
|
33128
|
+
}
|
|
32923
33129
|
}
|
|
32924
|
-
|
|
32925
|
-
}
|
|
32926
|
-
|
|
32927
|
-
|
|
32928
|
-
|
|
32929
|
-
|
|
32930
|
-
|
|
32931
|
-
|
|
32932
|
-
record_id: record.id,
|
|
32933
|
-
record_type: record.type,
|
|
32934
|
-
repository: index.source.repo,
|
|
32935
|
-
graph: index.source.graph ?? null,
|
|
32936
|
-
source: record.source,
|
|
32937
|
-
privacy: record.privacy
|
|
32938
|
-
} },
|
|
32939
|
-
format: "markdown",
|
|
32940
|
-
source: "aiwg-index",
|
|
32941
|
-
starred: false,
|
|
32942
|
-
archived: false,
|
|
32943
|
-
collection_id: null,
|
|
32944
|
-
created_at: sourceCreatedAt,
|
|
32945
|
-
updated_at: updatedAt,
|
|
32946
|
-
tags: recordTags,
|
|
32947
|
-
attachments: []
|
|
32948
|
-
});
|
|
32949
|
-
const hash = await sha256Hex(encoder4.encode(content));
|
|
32950
|
-
rows.get("note_originals").push({
|
|
32951
|
-
id: uuid("note-original", record.id),
|
|
32952
|
-
note_id: noteId,
|
|
32953
|
-
content,
|
|
32954
|
-
hash: `sha256:${hash}`,
|
|
32955
|
-
user_created_at: sourceCreatedAt,
|
|
32956
|
-
user_last_edited_at: updatedAt,
|
|
32957
|
-
version_number: 1
|
|
32958
|
-
});
|
|
32959
|
-
rows.get("note_revised_current").push({
|
|
32960
|
-
note_id: noteId,
|
|
32961
|
-
content,
|
|
32962
|
-
last_revision_id: null,
|
|
32963
|
-
ai_metadata: null
|
|
32964
|
-
});
|
|
32965
|
-
}
|
|
32966
|
-
rows.set("tags", [...tags].sort().map((name) => ({ name, created_at: exportedAt })));
|
|
32967
|
-
const relationshipInput = [];
|
|
32968
|
-
for (const record of records) {
|
|
32969
|
-
for (const [position, relationship] of record.relationships.entries()) {
|
|
32970
|
-
const target = noteIds.get(relationship.target_id) ?? null;
|
|
32971
|
-
const key = `${record.id}\0${position}\0${relationship.type}\0${relationship.target_id}`;
|
|
32972
|
-
rows.get("links").push({
|
|
32973
|
-
id: uuid("relationship", key),
|
|
32974
|
-
from_note_id: noteIds.get(record.id),
|
|
32975
|
-
to_note_id: target,
|
|
32976
|
-
to_url: target ? null : `aiwg://record/${encodeURIComponent(relationship.target_id)}`,
|
|
32977
|
-
kind: relationship.type,
|
|
32978
|
-
score: own(relationship, "confidence") ? relationship.confidence ?? null : null,
|
|
32979
|
-
created_at: new Date(record.updated_at).toISOString(),
|
|
32980
|
-
metadata: target ? relationship.metadata ?? null : {
|
|
32981
|
-
direction: relationship.direction ?? null,
|
|
32982
|
-
label: relationship.label ?? null,
|
|
32983
|
-
privacy: relationship.privacy ?? null,
|
|
32984
|
-
source_path: relationship.source_path ?? null,
|
|
32985
|
-
target_path: relationship.target_path ?? null,
|
|
32986
|
-
source_metadata: relationship.metadata ?? null
|
|
33130
|
+
}
|
|
33131
|
+
},
|
|
33132
|
+
{
|
|
33133
|
+
if: {
|
|
33134
|
+
properties: {
|
|
33135
|
+
schema_version: {
|
|
33136
|
+
const: "aiwg.fortemi.index.export.v2"
|
|
33137
|
+
}
|
|
32987
33138
|
}
|
|
32988
|
-
}
|
|
32989
|
-
|
|
32990
|
-
|
|
32991
|
-
|
|
32992
|
-
|
|
32993
|
-
|
|
32994
|
-
|
|
32995
|
-
|
|
32996
|
-
|
|
32997
|
-
|
|
32998
|
-
|
|
32999
|
-
|
|
33000
|
-
|
|
33001
|
-
|
|
33002
|
-
|
|
33003
|
-
destination_capability: "required-number",
|
|
33004
|
-
action: "default"
|
|
33139
|
+
},
|
|
33140
|
+
then: {
|
|
33141
|
+
required: ["compatibility"],
|
|
33142
|
+
properties: {
|
|
33143
|
+
source: {
|
|
33144
|
+
required: ["graph"]
|
|
33145
|
+
},
|
|
33146
|
+
items: {
|
|
33147
|
+
items: {
|
|
33148
|
+
properties: {
|
|
33149
|
+
schema_version: {
|
|
33150
|
+
const: "aiwg.fortemi.index.record.v2"
|
|
33151
|
+
}
|
|
33152
|
+
}
|
|
33153
|
+
}
|
|
33005
33154
|
}
|
|
33006
|
-
);
|
|
33007
|
-
}
|
|
33008
|
-
rows.get("graph_edges").push({
|
|
33009
|
-
graph_source_id: "aiwg-relationships",
|
|
33010
|
-
from_note_id: noteIds.get(record.id),
|
|
33011
|
-
to_note_id: target,
|
|
33012
|
-
weight,
|
|
33013
|
-
kind: "manual",
|
|
33014
|
-
rank: position,
|
|
33015
|
-
metadata: {
|
|
33016
|
-
type: relationship.type,
|
|
33017
|
-
direction: relationship.direction ?? null,
|
|
33018
|
-
label: relationship.label ?? null,
|
|
33019
|
-
privacy: relationship.privacy ?? null,
|
|
33020
|
-
source_path: relationship.source_path ?? null,
|
|
33021
|
-
target_path: relationship.target_path ?? null,
|
|
33022
|
-
source_metadata: relationship.metadata ?? null
|
|
33023
33155
|
}
|
|
33024
|
-
}
|
|
33025
|
-
}
|
|
33026
|
-
}
|
|
33027
|
-
if (rows.get("graph_edges").length) {
|
|
33028
|
-
rows.get("graph_sources").push({
|
|
33029
|
-
id: "aiwg-relationships",
|
|
33030
|
-
name: "AIWG relationships",
|
|
33031
|
-
kind: "imported",
|
|
33032
|
-
source_table: "manual",
|
|
33033
|
-
embedding_set_id: null,
|
|
33034
|
-
virtual_set_id: null,
|
|
33035
|
-
model: null,
|
|
33036
|
-
dimension: null,
|
|
33037
|
-
truncate_dimension: null,
|
|
33038
|
-
metric: null,
|
|
33039
|
-
algorithm: null,
|
|
33040
|
-
parameters: null,
|
|
33041
|
-
input_hash: `sha256:${await sha256Hex(encoder4.encode(JSON.stringify(relationshipInput)))}`,
|
|
33042
|
-
freshness: { status: "fresh", checked_at: exportedAt },
|
|
33043
|
-
created_at: exportedAt
|
|
33044
|
-
});
|
|
33045
|
-
}
|
|
33046
|
-
const concepts = /* @__PURE__ */ new Map();
|
|
33047
|
-
const conceptMembers = /* @__PURE__ */ new Map();
|
|
33048
|
-
for (const record of records) {
|
|
33049
|
-
for (const id of record.concepts) {
|
|
33050
|
-
conceptMembers.set(id, [...conceptMembers.get(id) ?? [], record]);
|
|
33156
|
+
}
|
|
33051
33157
|
}
|
|
33052
|
-
|
|
33053
|
-
|
|
33054
|
-
|
|
33055
|
-
|
|
33056
|
-
|
|
33057
|
-
|
|
33058
|
-
|
|
33158
|
+
],
|
|
33159
|
+
$defs: {
|
|
33160
|
+
privacy: {
|
|
33161
|
+
enum: ["private", "sanitized", "public"]
|
|
33162
|
+
},
|
|
33163
|
+
recordType: {
|
|
33164
|
+
type: "string",
|
|
33165
|
+
minLength: 1
|
|
33166
|
+
},
|
|
33167
|
+
stringArray: {
|
|
33168
|
+
type: "array",
|
|
33169
|
+
items: {
|
|
33170
|
+
type: "string"
|
|
33171
|
+
}
|
|
33172
|
+
},
|
|
33173
|
+
numberArray: {
|
|
33174
|
+
type: "array",
|
|
33175
|
+
items: {
|
|
33176
|
+
type: "number"
|
|
33177
|
+
}
|
|
33178
|
+
},
|
|
33179
|
+
v1RecordCompatibility: {
|
|
33180
|
+
type: "object",
|
|
33181
|
+
not: {
|
|
33182
|
+
anyOf: [
|
|
33059
33183
|
{
|
|
33060
|
-
|
|
33061
|
-
|
|
33062
|
-
|
|
33063
|
-
|
|
33184
|
+
required: ["name"]
|
|
33185
|
+
},
|
|
33186
|
+
{
|
|
33187
|
+
required: ["summary"]
|
|
33188
|
+
},
|
|
33189
|
+
{
|
|
33190
|
+
required: ["search"]
|
|
33191
|
+
},
|
|
33192
|
+
{
|
|
33193
|
+
required: ["chunks"]
|
|
33194
|
+
},
|
|
33195
|
+
{
|
|
33196
|
+
required: ["embeddings"]
|
|
33197
|
+
},
|
|
33198
|
+
{
|
|
33199
|
+
required: ["compatibility"]
|
|
33200
|
+
},
|
|
33201
|
+
{
|
|
33202
|
+
required: ["skos_concepts"]
|
|
33203
|
+
},
|
|
33204
|
+
{
|
|
33205
|
+
required: ["skos_relations"]
|
|
33206
|
+
},
|
|
33207
|
+
{
|
|
33208
|
+
required: ["provenance_events"]
|
|
33209
|
+
},
|
|
33210
|
+
{
|
|
33211
|
+
required: ["operational_state"]
|
|
33212
|
+
},
|
|
33213
|
+
{
|
|
33214
|
+
required: ["state_transfer"]
|
|
33064
33215
|
}
|
|
33065
|
-
|
|
33066
|
-
}
|
|
33067
|
-
|
|
33068
|
-
|
|
33069
|
-
|
|
33070
|
-
|
|
33071
|
-
|
|
33072
|
-
|
|
33073
|
-
|
|
33074
|
-
|
|
33075
|
-
|
|
33076
|
-
|
|
33077
|
-
|
|
33078
|
-
|
|
33079
|
-
|
|
33080
|
-
|
|
33081
|
-
|
|
33082
|
-
|
|
33083
|
-
|
|
33084
|
-
);
|
|
33085
|
-
}
|
|
33086
|
-
}
|
|
33087
|
-
const schemeIds = new Map(
|
|
33088
|
-
[...new Set([...concepts.values()].map(schemeName))].sort().map((name) => [name, uuid("skos-scheme", name)])
|
|
33089
|
-
);
|
|
33090
|
-
for (const [name, id] of schemeIds) {
|
|
33091
|
-
rows.get("skos_schemes").push({
|
|
33092
|
-
id,
|
|
33093
|
-
uri: null,
|
|
33094
|
-
notation: name,
|
|
33095
|
-
title: name,
|
|
33096
|
-
description: null,
|
|
33097
|
-
creator: "aiwg-index",
|
|
33098
|
-
publisher: null,
|
|
33099
|
-
rights: null,
|
|
33100
|
-
version: null,
|
|
33101
|
-
is_active: true,
|
|
33102
|
-
is_system: false,
|
|
33103
|
-
created_at: exportedAt,
|
|
33104
|
-
updated_at: exportedAt,
|
|
33105
|
-
issued_at: null,
|
|
33106
|
-
modified_at: null,
|
|
33107
|
-
embedding: null,
|
|
33108
|
-
embedding_model: null,
|
|
33109
|
-
embedded_at: null
|
|
33110
|
-
});
|
|
33111
|
-
}
|
|
33112
|
-
const conceptIds = new Map([...concepts.keys()].sort().map((id) => [id, uuid("skos-concept", id)]));
|
|
33113
|
-
for (const [sourceId, concept] of [...concepts].sort(([a], [b]) => a.localeCompare(b))) {
|
|
33114
|
-
const id = conceptIds.get(sourceId);
|
|
33115
|
-
const schemeId = schemeIds.get(schemeName(concept));
|
|
33116
|
-
const memberCount = conceptMembers.get(sourceId)?.length ?? 0;
|
|
33117
|
-
if (concept.uri && !safeUri(concept.uri)) addLoss(
|
|
33118
|
-
losses,
|
|
33119
|
-
"aiwg-skos-uri-invalid",
|
|
33120
|
-
"AIWG concept URI is not a valid authority URI and was mapped to null.",
|
|
33121
|
-
{
|
|
33122
|
-
component: "skos_concepts",
|
|
33123
|
-
record_id: sourceId,
|
|
33124
|
-
field_path: "/uri",
|
|
33125
|
-
source_state: "value",
|
|
33126
|
-
destination_capability: "uri",
|
|
33127
|
-
action: "omit"
|
|
33128
|
-
}
|
|
33129
|
-
);
|
|
33130
|
-
const unmappedConceptMetadata = Object.keys(concept.metadata ?? {}).filter((key) => key !== "domain");
|
|
33131
|
-
if (unmappedConceptMetadata.length > 0) addLoss(
|
|
33132
|
-
losses,
|
|
33133
|
-
"aiwg-skos-metadata-unmapped",
|
|
33134
|
-
"AIWG concept metadata without a native full-v1 field was omitted.",
|
|
33135
|
-
{
|
|
33136
|
-
component: "skos_concepts",
|
|
33137
|
-
record_id: sourceId,
|
|
33138
|
-
source_state: "value",
|
|
33139
|
-
action: "omit",
|
|
33140
|
-
destination_capability: "native-skos-fields",
|
|
33141
|
-
reason: unmappedConceptMetadata.join(",")
|
|
33142
|
-
}
|
|
33143
|
-
);
|
|
33144
|
-
rows.get("skos_concepts").push({
|
|
33145
|
-
id,
|
|
33146
|
-
primary_scheme_id: schemeId,
|
|
33147
|
-
uri: safeUri(concept.uri),
|
|
33148
|
-
notation: concept.notation ?? sourceId,
|
|
33149
|
-
facet_type: null,
|
|
33150
|
-
facet_source: "aiwg-index",
|
|
33151
|
-
facet_domain: typeof concept.metadata?.domain === "string" ? concept.metadata.domain : null,
|
|
33152
|
-
facet_scope: null,
|
|
33153
|
-
status: "candidate",
|
|
33154
|
-
promoted_at: null,
|
|
33155
|
-
deprecated_at: null,
|
|
33156
|
-
deprecation_reason: null,
|
|
33157
|
-
replaced_by_id: null,
|
|
33158
|
-
note_count: memberCount,
|
|
33159
|
-
first_used_at: memberCount ? exportedAt : null,
|
|
33160
|
-
last_used_at: memberCount ? exportedAt : null,
|
|
33161
|
-
depth: 0,
|
|
33162
|
-
broader_count: 0,
|
|
33163
|
-
narrower_count: 0,
|
|
33164
|
-
related_count: 0,
|
|
33165
|
-
antipatterns: null,
|
|
33166
|
-
antipattern_checked_at: null,
|
|
33167
|
-
created_at: exportedAt,
|
|
33168
|
-
updated_at: exportedAt,
|
|
33169
|
-
embedding: null,
|
|
33170
|
-
embedding_model: null,
|
|
33171
|
-
embedded_at: null
|
|
33172
|
-
});
|
|
33173
|
-
rows.get("skos_scheme_memberships").push({
|
|
33174
|
-
concept_id: id,
|
|
33175
|
-
scheme_id: schemeId,
|
|
33176
|
-
is_top_concept: true,
|
|
33177
|
-
added_at: exportedAt
|
|
33178
|
-
});
|
|
33179
|
-
rows.get("skos_labels").push({
|
|
33180
|
-
id: uuid("skos-label", `${sourceId}:pref:${concept.prefLabel}`),
|
|
33181
|
-
concept_id: id,
|
|
33182
|
-
label_type: "pref_label",
|
|
33183
|
-
value: concept.prefLabel,
|
|
33184
|
-
language: "und",
|
|
33185
|
-
created_at: exportedAt
|
|
33186
|
-
});
|
|
33187
|
-
for (const label of concept.altLabels ?? []) rows.get("skos_labels").push({
|
|
33188
|
-
id: uuid("skos-label", `${sourceId}:alt:${label}`),
|
|
33189
|
-
concept_id: id,
|
|
33190
|
-
label_type: "alt_label",
|
|
33191
|
-
value: label,
|
|
33192
|
-
language: "und",
|
|
33193
|
-
created_at: exportedAt
|
|
33194
|
-
});
|
|
33195
|
-
if (concept.definition !== void 0) rows.get("skos_notes").push({
|
|
33196
|
-
id: uuid("skos-note", `${sourceId}:definition`),
|
|
33197
|
-
concept_id: id,
|
|
33198
|
-
note_type: "definition",
|
|
33199
|
-
value: concept.definition,
|
|
33200
|
-
language: "und",
|
|
33201
|
-
author: null,
|
|
33202
|
-
source: concept.uri ?? null,
|
|
33203
|
-
created_at: exportedAt,
|
|
33204
|
-
updated_at: exportedAt
|
|
33205
|
-
});
|
|
33206
|
-
}
|
|
33207
|
-
for (const record of records) {
|
|
33208
|
-
for (const [position, sourceId] of record.concepts.entries()) {
|
|
33209
|
-
const conceptId = conceptIds.get(sourceId);
|
|
33210
|
-
if (conceptId) rows.get("note_skos_tags").push({
|
|
33211
|
-
note_id: noteIds.get(record.id),
|
|
33212
|
-
concept_id: conceptId,
|
|
33213
|
-
source: "aiwg-index",
|
|
33214
|
-
confidence: null,
|
|
33215
|
-
relevance_score: null,
|
|
33216
|
-
is_primary: position === 0,
|
|
33217
|
-
created_at: new Date(record.updated_at).toISOString(),
|
|
33218
|
-
created_by: "aiwg-index"
|
|
33219
|
-
});
|
|
33220
|
-
}
|
|
33221
|
-
for (const relation of record.skos_relations ?? []) {
|
|
33222
|
-
const subject = conceptIds.get(relation.source_id);
|
|
33223
|
-
const object = conceptIds.get(relation.target_id);
|
|
33224
|
-
if (!subject || !object || !["broader", "narrower", "related"].includes(relation.type)) {
|
|
33225
|
-
addLoss(
|
|
33226
|
-
losses,
|
|
33227
|
-
"aiwg-skos-relation-unrepresentable",
|
|
33228
|
-
"SKOS relation could not be mapped to the native full-v1 relation enum.",
|
|
33229
|
-
{
|
|
33230
|
-
component: "skos_relations",
|
|
33231
|
-
record_id: record.id,
|
|
33232
|
-
destination_capability: "broader-narrower-related",
|
|
33233
|
-
action: "omit"
|
|
33216
|
+
]
|
|
33217
|
+
},
|
|
33218
|
+
properties: {
|
|
33219
|
+
source: {
|
|
33220
|
+
not: {
|
|
33221
|
+
anyOf: [
|
|
33222
|
+
{
|
|
33223
|
+
required: ["origin"]
|
|
33224
|
+
},
|
|
33225
|
+
{
|
|
33226
|
+
required: ["generated"]
|
|
33227
|
+
},
|
|
33228
|
+
{
|
|
33229
|
+
required: ["checksum"]
|
|
33230
|
+
},
|
|
33231
|
+
{
|
|
33232
|
+
required: ["updated_at"]
|
|
33233
|
+
}
|
|
33234
|
+
]
|
|
33234
33235
|
}
|
|
33235
|
-
|
|
33236
|
-
|
|
33237
|
-
|
|
33238
|
-
|
|
33239
|
-
|
|
33240
|
-
|
|
33241
|
-
|
|
33242
|
-
|
|
33243
|
-
|
|
33244
|
-
|
|
33245
|
-
|
|
33246
|
-
|
|
33247
|
-
|
|
33248
|
-
|
|
33249
|
-
|
|
33250
|
-
|
|
33251
|
-
|
|
33252
|
-
|
|
33253
|
-
|
|
33236
|
+
},
|
|
33237
|
+
relationships: {
|
|
33238
|
+
items: {
|
|
33239
|
+
not: {
|
|
33240
|
+
anyOf: [
|
|
33241
|
+
{
|
|
33242
|
+
required: ["target_path"]
|
|
33243
|
+
},
|
|
33244
|
+
{
|
|
33245
|
+
required: ["direction"]
|
|
33246
|
+
},
|
|
33247
|
+
{
|
|
33248
|
+
required: ["label"]
|
|
33249
|
+
},
|
|
33250
|
+
{
|
|
33251
|
+
required: ["confidence"]
|
|
33252
|
+
},
|
|
33253
|
+
{
|
|
33254
|
+
required: ["privacy"]
|
|
33255
|
+
},
|
|
33256
|
+
{
|
|
33257
|
+
required: ["metadata"]
|
|
33258
|
+
}
|
|
33259
|
+
]
|
|
33260
|
+
}
|
|
33254
33261
|
}
|
|
33255
|
-
|
|
33256
|
-
|
|
33257
|
-
|
|
33258
|
-
|
|
33259
|
-
id: uuid("skos-relation", `${relation.source_id}:${relation.type}:${relation.target_id}`),
|
|
33260
|
-
subject_id: subject,
|
|
33261
|
-
object_id: object,
|
|
33262
|
-
relation_type: relation.type,
|
|
33263
|
-
inference_score: null,
|
|
33264
|
-
is_inferred: false,
|
|
33265
|
-
is_validated: false,
|
|
33266
|
-
created_at: new Date(record.updated_at).toISOString(),
|
|
33267
|
-
created_by: "aiwg-index"
|
|
33268
|
-
});
|
|
33269
|
-
}
|
|
33270
|
-
}
|
|
33271
|
-
for (const record of records) {
|
|
33272
|
-
for (const [position, event] of (record.provenance_events ?? []).entries()) {
|
|
33273
|
-
const start = timestamp(event.started_at);
|
|
33274
|
-
if (!start) addLoss(
|
|
33275
|
-
losses,
|
|
33276
|
-
"aiwg-provenance-start-unavailable",
|
|
33277
|
-
"Activity start defaulted to record updated_at.",
|
|
33278
|
-
{
|
|
33279
|
-
component: "provenance_activities",
|
|
33280
|
-
record_id: event.id ?? `${record.id}:${position}`,
|
|
33281
|
-
field_path: "/started_at",
|
|
33282
|
-
source_state: "absent",
|
|
33283
|
-
action: "default"
|
|
33284
|
-
}
|
|
33285
|
-
);
|
|
33286
|
-
rows.get("provenance_activities").push({
|
|
33287
|
-
id: uuid("provenance-event", event.id ?? `${record.id}:${position}`),
|
|
33288
|
-
note_id: noteIds.get(record.id),
|
|
33289
|
-
revision_id: null,
|
|
33290
|
-
activity_type: event.activity,
|
|
33291
|
-
model_name: event.agent ?? null,
|
|
33292
|
-
started_at: start ?? new Date(record.updated_at).toISOString(),
|
|
33293
|
-
ended_at: timestamp(event.ended_at) ?? null,
|
|
33294
|
-
metadata: {
|
|
33295
|
-
source: event.source ?? null,
|
|
33296
|
-
path: event.path ?? null,
|
|
33297
|
-
confidence: event.confidence ?? null,
|
|
33298
|
-
privacy: event.privacy ?? null,
|
|
33299
|
-
attributes: event.attributes ?? null
|
|
33300
|
-
}
|
|
33301
|
-
});
|
|
33302
|
-
}
|
|
33303
|
-
for (const [position, provenance] of record.provenance.entries()) {
|
|
33304
|
-
rows.get("provenance_activities").push({
|
|
33305
|
-
id: uuid("provenance-field", `${record.id}:${position}:${provenance.field}`),
|
|
33306
|
-
note_id: noteIds.get(record.id),
|
|
33307
|
-
revision_id: null,
|
|
33308
|
-
activity_type: `source:${provenance.field}`,
|
|
33309
|
-
model_name: provenance.source,
|
|
33310
|
-
started_at: new Date(record.updated_at).toISOString(),
|
|
33311
|
-
ended_at: null,
|
|
33312
|
-
metadata: { path: provenance.path, confidence: provenance.confidence, privacy: provenance.privacy }
|
|
33313
|
-
});
|
|
33314
|
-
}
|
|
33315
|
-
}
|
|
33316
|
-
const embeddingGroups = /* @__PURE__ */ new Map();
|
|
33317
|
-
for (const record of records) {
|
|
33318
|
-
for (const [position, embedding] of (record.embeddings ?? []).entries()) {
|
|
33319
|
-
const vector = embedding.embedding ?? embedding.vector;
|
|
33320
|
-
if (!vector || vector.length !== 768) {
|
|
33321
|
-
addLoss(
|
|
33322
|
-
losses,
|
|
33323
|
-
"aiwg-embedding-dimension-unrepresentable",
|
|
33324
|
-
"Only 768-dimensional vectors satisfy the full-v1 authority.",
|
|
33325
|
-
{
|
|
33326
|
-
component: "embeddings",
|
|
33327
|
-
record_id: embedding.id ?? `${record.id}:${position}`,
|
|
33328
|
-
field_path: "/vector",
|
|
33329
|
-
source_state: vector ? "value" : "absent",
|
|
33330
|
-
destination_capability: "768-dimensional-vector",
|
|
33331
|
-
action: "omit"
|
|
33332
|
-
}
|
|
33333
|
-
);
|
|
33334
|
-
continue;
|
|
33335
|
-
}
|
|
33336
|
-
const model = embedding.model ?? "aiwg-unknown";
|
|
33337
|
-
if (!embedding.model) addLoss(
|
|
33338
|
-
losses,
|
|
33339
|
-
"aiwg-embedding-model-unavailable",
|
|
33340
|
-
"Embedding model defaulted because AIWG did not supply one.",
|
|
33341
|
-
{
|
|
33342
|
-
component: "embeddings",
|
|
33343
|
-
record_id: embedding.id ?? `${record.id}:${position}`,
|
|
33344
|
-
field_path: "/model",
|
|
33345
|
-
source_state: "absent",
|
|
33346
|
-
action: "default"
|
|
33347
|
-
}
|
|
33348
|
-
);
|
|
33349
|
-
let group = embeddingGroups.get(model);
|
|
33350
|
-
if (!group) {
|
|
33351
|
-
group = { config: uuid("embedding-config", model), set: uuid("embedding-set", model), count: 0 };
|
|
33352
|
-
embeddingGroups.set(model, group);
|
|
33353
|
-
}
|
|
33354
|
-
group.count += 1;
|
|
33355
|
-
const unmappedEmbeddingFields = [
|
|
33356
|
-
embedding.granularity !== void 0 ? "granularity" : null,
|
|
33357
|
-
embedding.source_path !== void 0 ? "source_path" : null,
|
|
33358
|
-
embedding.metadata !== void 0 ? "metadata" : null,
|
|
33359
|
-
embedding.input_hash !== void 0 && !/^[0-9a-f]{64}$/.test(embedding.input_hash) ? "input_hash" : null
|
|
33360
|
-
].filter((field) => field !== null);
|
|
33361
|
-
if (unmappedEmbeddingFields.length > 0) addLoss(
|
|
33362
|
-
losses,
|
|
33363
|
-
"aiwg-embedding-fields-unmapped",
|
|
33364
|
-
"AIWG embedding fields without a native full-v1 field were omitted.",
|
|
33365
|
-
{
|
|
33366
|
-
component: "embeddings",
|
|
33367
|
-
record_id: embedding.id ?? `${record.id}:${position}`,
|
|
33368
|
-
source_state: "value",
|
|
33369
|
-
action: "omit",
|
|
33370
|
-
destination_capability: "native-embedding-fields",
|
|
33371
|
-
reason: unmappedEmbeddingFields.join(",")
|
|
33372
|
-
}
|
|
33373
|
-
);
|
|
33374
|
-
rows.get("embeddings").push({
|
|
33375
|
-
id: uuid("embedding", embedding.id ?? `${record.id}:${position}`),
|
|
33376
|
-
note_id: noteIds.get(record.id),
|
|
33377
|
-
embedding_set_id: group.set,
|
|
33378
|
-
chunk_index: position,
|
|
33379
|
-
text: record.text ?? "",
|
|
33380
|
-
vector,
|
|
33381
|
-
model,
|
|
33382
|
-
contract_fingerprint: /^[0-9a-f]{64}$/.test(embedding.input_hash ?? "") ? embedding.input_hash : null,
|
|
33383
|
-
created_at: new Date(record.updated_at).toISOString()
|
|
33384
|
-
});
|
|
33385
|
-
}
|
|
33386
|
-
}
|
|
33387
|
-
for (const [model, group] of [...embeddingGroups].sort(([a], [b]) => a.localeCompare(b))) {
|
|
33388
|
-
rows.get("embedding_configs").push({
|
|
33389
|
-
id: group.config,
|
|
33390
|
-
name: model,
|
|
33391
|
-
description: null,
|
|
33392
|
-
model,
|
|
33393
|
-
dimension: 768,
|
|
33394
|
-
chunk_size: 1,
|
|
33395
|
-
chunk_overlap: 0,
|
|
33396
|
-
hnsw_m: null,
|
|
33397
|
-
hnsw_ef_construction: null,
|
|
33398
|
-
ivfflat_lists: null,
|
|
33399
|
-
is_default: null,
|
|
33400
|
-
supports_mrl: null,
|
|
33401
|
-
matryoshka_dims: null,
|
|
33402
|
-
default_truncate_dim: null,
|
|
33403
|
-
provider: null,
|
|
33404
|
-
provider_config: null,
|
|
33405
|
-
content_types: null,
|
|
33406
|
-
strengths: null,
|
|
33407
|
-
limitations: null,
|
|
33408
|
-
recommended_for: null,
|
|
33409
|
-
benchmark_scores: null,
|
|
33410
|
-
is_available: null,
|
|
33411
|
-
document_composition: {},
|
|
33412
|
-
created_at: exportedAt,
|
|
33413
|
-
updated_at: exportedAt
|
|
33414
|
-
});
|
|
33415
|
-
rows.get("embedding_sets").push({
|
|
33416
|
-
id: group.set,
|
|
33417
|
-
name: model,
|
|
33418
|
-
slug: uuid("embedding-slug", model),
|
|
33419
|
-
description: null,
|
|
33420
|
-
purpose: null,
|
|
33421
|
-
usage_hints: null,
|
|
33422
|
-
keywords: null,
|
|
33423
|
-
set_type: "full",
|
|
33424
|
-
mode: "manual",
|
|
33425
|
-
criteria: null,
|
|
33426
|
-
embedding_config_id: group.config,
|
|
33427
|
-
truncate_dim: null,
|
|
33428
|
-
auto_embed_rules: null,
|
|
33429
|
-
index_status: "ready",
|
|
33430
|
-
index_type: null,
|
|
33431
|
-
last_indexed_at: exportedAt,
|
|
33432
|
-
document_count: null,
|
|
33433
|
-
embedding_count: group.count,
|
|
33434
|
-
embeddings_current: true,
|
|
33435
|
-
index_size_bytes: null,
|
|
33436
|
-
is_system: false,
|
|
33437
|
-
is_active: true,
|
|
33438
|
-
auto_refresh: false,
|
|
33439
|
-
refresh_interval: null,
|
|
33440
|
-
last_refresh_at: null,
|
|
33441
|
-
agent_metadata: null,
|
|
33442
|
-
created_at: exportedAt,
|
|
33443
|
-
updated_at: exportedAt,
|
|
33444
|
-
created_by: "aiwg-index"
|
|
33445
|
-
});
|
|
33446
|
-
const noteIdSet = new Set(rows.get("embeddings").filter((item) => item.embedding_set_id === group.set).map((item) => item.note_id));
|
|
33447
|
-
for (const noteId of [...noteIdSet].sort()) rows.get("embedding_set_members").push({
|
|
33448
|
-
embedding_set_id: group.set,
|
|
33449
|
-
note_id: noteId,
|
|
33450
|
-
membership_type: "materialized",
|
|
33451
|
-
added_at: exportedAt,
|
|
33452
|
-
added_by: "aiwg-index"
|
|
33453
|
-
});
|
|
33454
|
-
}
|
|
33455
|
-
if (losses.length > 0) {
|
|
33456
|
-
return {
|
|
33457
|
-
success: false,
|
|
33458
|
-
archive: null,
|
|
33459
|
-
profile: "full-v1",
|
|
33460
|
-
schema_version: "2.0.0",
|
|
33461
|
-
lossless: false,
|
|
33462
|
-
losses,
|
|
33463
|
-
receipt: {
|
|
33464
|
-
schema_version: "fortemi.aiwg-full-v1-conversion-receipt.v1",
|
|
33465
|
-
source_schema_version: "aiwg.fortemi.index.export.v2",
|
|
33466
|
-
authority_repository: knowledge_shard_v2_schema_receipt_default.source.repository,
|
|
33467
|
-
authority_commit: knowledge_shard_v2_schema_receipt_default.source.commit,
|
|
33468
|
-
authority_contract_sha256: knowledge_shard_v2_schema_receipt_default.source.contractSha256,
|
|
33469
|
-
authority_schema_bundle_sha256: knowledge_shard_v2_schema_receipt_default.schemaBundle.sha256,
|
|
33470
|
-
manifest_sha256: null,
|
|
33471
|
-
component_checksums: {},
|
|
33472
|
-
contract_valid: false,
|
|
33473
|
-
signed: false
|
|
33474
|
-
}
|
|
33475
|
-
};
|
|
33476
|
-
}
|
|
33477
|
-
const files = /* @__PURE__ */ new Map();
|
|
33478
|
-
const components = Object.keys(FULL_V1_COMPONENT_FILES);
|
|
33479
|
-
const counts = { community_sets: 0 };
|
|
33480
|
-
for (const component of components) {
|
|
33481
|
-
const spec = FULL_V1_COMPONENT_FILES[component];
|
|
33482
|
-
const componentRows = rows.get(component) ?? [];
|
|
33483
|
-
files.set(spec.file, encode(componentRows, spec.encoding));
|
|
33484
|
-
counts[component] = componentRows.length;
|
|
33485
|
-
}
|
|
33486
|
-
const checksums = {};
|
|
33487
|
-
for (const [file, bytes] of files) checksums[file] = await sha256Hex(bytes);
|
|
33488
|
-
const manifest = {
|
|
33489
|
-
version: "2.0.0",
|
|
33490
|
-
profile: "full-v1",
|
|
33491
|
-
producer: { name: "fortemi-core-aiwg-index", version: options.matricVersion ?? "fortemi-core" },
|
|
33492
|
-
format: SHARD_FORMAT,
|
|
33493
|
-
created_at: exportedAt,
|
|
33494
|
-
components,
|
|
33495
|
-
counts,
|
|
33496
|
-
checksums,
|
|
33497
|
-
min_reader_version: "2.0.0"
|
|
33498
|
-
};
|
|
33499
|
-
const manifestBytes = encoder4.encode(JSON.stringify(manifest, null, 2));
|
|
33500
|
-
files.set("manifest.json", manifestBytes);
|
|
33501
|
-
const validation = await validateFullV1ShardArchive(files);
|
|
33502
|
-
if (!validation.valid) {
|
|
33503
|
-
throw new Error(`Generated AIWG full-v1 shard failed authority validation: ${validation.errors.join("; ")}`);
|
|
33504
|
-
}
|
|
33505
|
-
return {
|
|
33506
|
-
success: true,
|
|
33507
|
-
archive: packTarGz(files),
|
|
33508
|
-
profile: "full-v1",
|
|
33509
|
-
schema_version: "2.0.0",
|
|
33510
|
-
lossless: losses.length === 0,
|
|
33511
|
-
losses,
|
|
33512
|
-
receipt: {
|
|
33513
|
-
schema_version: "fortemi.aiwg-full-v1-conversion-receipt.v1",
|
|
33514
|
-
source_schema_version: "aiwg.fortemi.index.export.v2",
|
|
33515
|
-
authority_repository: knowledge_shard_v2_schema_receipt_default.source.repository,
|
|
33516
|
-
authority_commit: knowledge_shard_v2_schema_receipt_default.source.commit,
|
|
33517
|
-
authority_contract_sha256: knowledge_shard_v2_schema_receipt_default.source.contractSha256,
|
|
33518
|
-
authority_schema_bundle_sha256: knowledge_shard_v2_schema_receipt_default.schemaBundle.sha256,
|
|
33519
|
-
manifest_sha256: await sha256Hex(manifestBytes),
|
|
33520
|
-
component_checksums: checksums,
|
|
33521
|
-
contract_valid: true,
|
|
33522
|
-
signed: false
|
|
33523
|
-
}
|
|
33524
|
-
};
|
|
33525
|
-
}
|
|
33526
|
-
|
|
33527
|
-
// src/aiwg-index-shard.ts
|
|
33528
|
-
var AIWG_SHARD_UUID_NAMESPACE = "7ab5d1f8-29d2-5e35-9e2f-3a45de171a9e";
|
|
33529
|
-
var shardEncoder = new TextEncoder();
|
|
33530
|
-
var shardDecoder = new TextDecoder();
|
|
33531
|
-
function aiwgShardUuid(kind, id) {
|
|
33532
|
-
return v5(`${kind}:${id}`, AIWG_SHARD_UUID_NAMESPACE);
|
|
33533
|
-
}
|
|
33534
|
-
function aiwgShardTimestamp(value, fallback) {
|
|
33535
|
-
if (!value || Number.isNaN(Date.parse(value))) return fallback;
|
|
33536
|
-
return new Date(value).toISOString();
|
|
33537
|
-
}
|
|
33538
|
-
function aiwgRecordTitle(record) {
|
|
33539
|
-
return record.title ?? record.search?.title ?? record.search?.name ?? record.id;
|
|
33540
|
-
}
|
|
33541
|
-
function aiwgRecordContent(record) {
|
|
33542
|
-
if (record.text) return record.text;
|
|
33543
|
-
if (record.search?.body) return record.search.body;
|
|
33544
|
-
const chunks = record.chunks?.map((chunk) => chunk.text ?? chunk.body ?? chunk.summary ?? "").filter(Boolean);
|
|
33545
|
-
if (chunks?.length) return chunks.join("\n\n");
|
|
33546
|
-
return record.search?.summary ?? "";
|
|
33547
|
-
}
|
|
33548
|
-
function aiwgShardMetadata(index, record) {
|
|
33549
|
-
return {
|
|
33550
|
-
aiwg_fortemi_index: {
|
|
33551
|
-
envelope: {
|
|
33552
|
-
schema_version: index.schema_version,
|
|
33553
|
-
generated_at: index.generated_at,
|
|
33554
|
-
source: index.source,
|
|
33555
|
-
...index.compatibility ? { compatibility: index.compatibility } : {}
|
|
33556
|
-
},
|
|
33557
|
-
record
|
|
33558
|
-
}
|
|
33559
|
-
};
|
|
33560
|
-
}
|
|
33561
|
-
function encodeJsonLines(values) {
|
|
33562
|
-
return shardEncoder.encode(values.map((value) => JSON.stringify(value)).join("\n"));
|
|
33563
|
-
}
|
|
33564
|
-
async function aiwgFortemiIndexToKnowledgeShard(index, options = {}) {
|
|
33565
|
-
if (options.includeNativeRichComponents) {
|
|
33566
|
-
throw new Error(
|
|
33567
|
-
"Native AIWG full-v1 conversion requires aiwgFortemiIndexToKnowledgeShardWithReport so semantic losses cannot be discarded"
|
|
33568
|
-
);
|
|
33569
|
-
}
|
|
33570
|
-
const validation = validateAiwgFortemiIndexExport(index);
|
|
33571
|
-
if (!validation.valid) {
|
|
33572
|
-
throw new Error(`Invalid AIWG Fortemi index export:
|
|
33573
|
-
${validation.errors.join("\n")}`);
|
|
33574
|
-
}
|
|
33575
|
-
if (index.schema_version !== "aiwg.fortemi.index.export.v2") {
|
|
33576
|
-
throw new Error("Knowledge Shard conversion requires aiwg.fortemi.index.export.v2");
|
|
33577
|
-
}
|
|
33578
|
-
const createdAt2 = aiwgShardTimestamp(options.createdAt ?? index.generated_at, (/* @__PURE__ */ new Date()).toISOString());
|
|
33579
|
-
const noteIds = new Map(index.items.map((record) => [record.id, aiwgShardUuid("record", record.id)]));
|
|
33580
|
-
const notes = index.items.map((record) => {
|
|
33581
|
-
const content = aiwgRecordContent(record);
|
|
33582
|
-
return {
|
|
33583
|
-
id: noteIds.get(record.id),
|
|
33584
|
-
title: aiwgRecordTitle(record),
|
|
33585
|
-
original_content: content,
|
|
33586
|
-
revised_content: content,
|
|
33587
|
-
metadata: aiwgShardMetadata(index, record),
|
|
33588
|
-
collection_id: null,
|
|
33589
|
-
attachments: [],
|
|
33590
|
-
format: "markdown",
|
|
33591
|
-
source: "aiwg-index",
|
|
33592
|
-
starred: false,
|
|
33593
|
-
archived: false,
|
|
33594
|
-
tags: [...new Set(record.tags)].sort(),
|
|
33595
|
-
created_at: aiwgShardTimestamp(record.source.updated_at ?? record.updated_at, createdAt2),
|
|
33596
|
-
updated_at: aiwgShardTimestamp(record.updated_at, createdAt2),
|
|
33597
|
-
deleted_at: null
|
|
33598
|
-
};
|
|
33599
|
-
});
|
|
33600
|
-
const links = [];
|
|
33601
|
-
for (const record of index.items) {
|
|
33602
|
-
for (const [position, relationship] of record.relationships.entries()) {
|
|
33603
|
-
const targetNoteId = noteIds.get(relationship.target_id) ?? null;
|
|
33604
|
-
links.push({
|
|
33605
|
-
id: aiwgShardUuid(
|
|
33606
|
-
"relationship",
|
|
33607
|
-
`${record.id}\0${position}\0${relationship.type}\0${relationship.target_id}`
|
|
33608
|
-
),
|
|
33609
|
-
from_note_id: noteIds.get(record.id),
|
|
33610
|
-
to_note_id: targetNoteId,
|
|
33611
|
-
to_url: targetNoteId ? null : `aiwg://record/${encodeURIComponent(relationship.target_id)}`,
|
|
33612
|
-
kind: relationship.type,
|
|
33613
|
-
score: relationship.confidence ?? 1,
|
|
33614
|
-
created_at: aiwgShardTimestamp(record.updated_at, createdAt2),
|
|
33615
|
-
metadata: {
|
|
33616
|
-
aiwg_fortemi_index: {
|
|
33617
|
-
source_record_id: record.id,
|
|
33618
|
-
target_record_id: relationship.target_id,
|
|
33619
|
-
relationship
|
|
33620
|
-
}
|
|
33621
|
-
}
|
|
33622
|
-
});
|
|
33623
|
-
}
|
|
33624
|
-
}
|
|
33625
|
-
const files = /* @__PURE__ */ new Map();
|
|
33626
|
-
const components = ["notes", "tags"];
|
|
33627
|
-
const counts = {
|
|
33628
|
-
notes: notes.length,
|
|
33629
|
-
tags: [...new Set(notes.flatMap((note) => note.tags))].length
|
|
33630
|
-
};
|
|
33631
|
-
files.set("notes.jsonl", encodeJsonLines(notes));
|
|
33632
|
-
files.set("tags.json", shardEncoder.encode(JSON.stringify(
|
|
33633
|
-
[...new Set(notes.flatMap((note) => note.tags))].sort().map((name) => ({ name, created_at: createdAt2 }))
|
|
33634
|
-
)));
|
|
33635
|
-
const addComponent = (component, filename, values, jsonLines = true) => {
|
|
33636
|
-
if (values.length === 0) return;
|
|
33637
|
-
components.push(component);
|
|
33638
|
-
counts[component] = values.length;
|
|
33639
|
-
files.set(filename, jsonLines ? encodeJsonLines(values) : shardEncoder.encode(JSON.stringify(values)));
|
|
33640
|
-
};
|
|
33641
|
-
addComponent("links", "links.jsonl", links);
|
|
33642
|
-
const checksums = {};
|
|
33643
|
-
for (const [filename, bytes] of files) checksums[filename] = await sha256Hex(bytes);
|
|
33644
|
-
const manifest = {
|
|
33645
|
-
version: CURRENT_SHARD_VERSION,
|
|
33646
|
-
profile: "core-v1",
|
|
33647
|
-
producer: {
|
|
33648
|
-
name: "fortemi-core-aiwg-index",
|
|
33649
|
-
version: options.matricVersion ?? "fortemi-core"
|
|
33650
|
-
},
|
|
33651
|
-
format: SHARD_FORMAT,
|
|
33652
|
-
created_at: createdAt2,
|
|
33653
|
-
components,
|
|
33654
|
-
counts: {
|
|
33655
|
-
notes: 0,
|
|
33656
|
-
collections: 0,
|
|
33657
|
-
tags: 0,
|
|
33658
|
-
templates: 0,
|
|
33659
|
-
links: 0,
|
|
33660
|
-
embedding_sets: 0,
|
|
33661
|
-
embedding_set_members: 0,
|
|
33662
|
-
embeddings: 0,
|
|
33663
|
-
embedding_configs: 0,
|
|
33664
|
-
...counts
|
|
33665
|
-
},
|
|
33666
|
-
checksums,
|
|
33667
|
-
min_reader_version: CURRENT_SHARD_VERSION
|
|
33668
|
-
};
|
|
33669
|
-
files.set("manifest.json", shardEncoder.encode(JSON.stringify(manifest, null, 2)));
|
|
33670
|
-
const contractValidation = await validateCoreV1ShardArchive(files);
|
|
33671
|
-
if (!contractValidation.valid) {
|
|
33672
|
-
throw new Error(
|
|
33673
|
-
`Generated AIWG core-v1 shard failed canonical validation: ${contractValidation.errors.join("; ")}`
|
|
33674
|
-
);
|
|
33675
|
-
}
|
|
33676
|
-
return packTarGz(files);
|
|
33677
|
-
}
|
|
33678
|
-
async function aiwgFortemiIndexToKnowledgeShardWithReport(index, options = {}) {
|
|
33679
|
-
const validation = validateAiwgFortemiIndexExport(index);
|
|
33680
|
-
if (!validation.valid) {
|
|
33681
|
-
throw new Error(`Invalid AIWG Fortemi index export:
|
|
33682
|
-
${validation.errors.join("\n")}`);
|
|
33683
|
-
}
|
|
33684
|
-
if (index.schema_version !== "aiwg.fortemi.index.export.v2") {
|
|
33685
|
-
throw new Error("Full-v1 Knowledge Shard conversion requires aiwg.fortemi.index.export.v2");
|
|
33686
|
-
}
|
|
33687
|
-
return convertAiwgIndexToFullV1(index, options);
|
|
33688
|
-
}
|
|
33689
|
-
function aiwgFortemiIndexFromKnowledgeShard(bytes) {
|
|
33690
|
-
const files = unpackTarGz(bytes);
|
|
33691
|
-
const noteBytes = files.get("notes.jsonl");
|
|
33692
|
-
const notes = (noteBytes ? shardDecoder.decode(noteBytes) : "").split("\n").filter(Boolean).map((line) => JSON.parse(line));
|
|
33693
|
-
if (notes.length === 0) throw new Error("Knowledge Shard contains no AIWG index notes");
|
|
33694
|
-
let envelope;
|
|
33695
|
-
const items = [];
|
|
33696
|
-
for (const note of notes) {
|
|
33697
|
-
const metadata = note.metadata && typeof note.metadata === "object" && !Array.isArray(note.metadata) ? note.metadata.aiwg_fortemi_index : void 0;
|
|
33698
|
-
if (!metadata || typeof metadata !== "object" || Array.isArray(metadata)) {
|
|
33699
|
-
throw new Error(`Knowledge Shard note ${note.id} has no AIWG index metadata`);
|
|
33700
|
-
}
|
|
33701
|
-
const value = metadata;
|
|
33702
|
-
envelope ??= value.envelope;
|
|
33703
|
-
items.push(value.record);
|
|
33704
|
-
}
|
|
33705
|
-
const restored = {
|
|
33706
|
-
...envelope,
|
|
33707
|
-
items: items.sort((left, right) => left.id.localeCompare(right.id))
|
|
33708
|
-
};
|
|
33709
|
-
const validation = validateAiwgFortemiIndexExport(restored);
|
|
33710
|
-
if (!validation.valid) {
|
|
33711
|
-
throw new Error(`Knowledge Shard contains invalid AIWG index metadata:
|
|
33712
|
-
${validation.errors.join("\n")}`);
|
|
33713
|
-
}
|
|
33714
|
-
return restored;
|
|
33715
|
-
}
|
|
33716
|
-
|
|
33717
|
-
// schemas/aiwg-fortemi-index-export.schema.json
|
|
33718
|
-
var aiwg_fortemi_index_export_schema_default = {
|
|
33719
|
-
$schema: "https://json-schema.org/draft/2020-12/schema",
|
|
33720
|
-
$id: "https://aiwg.io/schemas/aiwg-fortemi-index-export.json",
|
|
33721
|
-
title: "AIWG Fortemi Index Export",
|
|
33722
|
-
type: "object",
|
|
33723
|
-
additionalProperties: false,
|
|
33724
|
-
required: ["schema_version", "generated_at", "source", "items"],
|
|
33725
|
-
properties: {
|
|
33726
|
-
schema_version: {
|
|
33727
|
-
enum: ["aiwg.fortemi.index.export.v1", "aiwg.fortemi.index.export.v2"]
|
|
33728
|
-
},
|
|
33729
|
-
generated_at: {
|
|
33730
|
-
type: "string",
|
|
33731
|
-
format: "date-time"
|
|
33732
|
-
},
|
|
33733
|
-
source: {
|
|
33734
|
-
type: "object",
|
|
33735
|
-
additionalProperties: false,
|
|
33736
|
-
required: ["repo", "privacy"],
|
|
33737
|
-
properties: {
|
|
33738
|
-
repo: {
|
|
33739
|
-
type: "string",
|
|
33740
|
-
minLength: 1
|
|
33741
|
-
},
|
|
33742
|
-
privacy: {
|
|
33743
|
-
$ref: "#/$defs/privacy"
|
|
33744
|
-
},
|
|
33745
|
-
graph: {
|
|
33746
|
-
type: "string",
|
|
33747
|
-
minLength: 1
|
|
33748
|
-
}
|
|
33749
|
-
}
|
|
33750
|
-
},
|
|
33751
|
-
compatibility: {
|
|
33752
|
-
type: "object",
|
|
33753
|
-
additionalProperties: false,
|
|
33754
|
-
required: ["previous_schema_version", "strategy"],
|
|
33755
|
-
properties: {
|
|
33756
|
-
previous_schema_version: {
|
|
33757
|
-
const: "aiwg.fortemi.index.export.v1"
|
|
33758
|
-
},
|
|
33759
|
-
strategy: {
|
|
33760
|
-
const: "supported"
|
|
33761
|
-
}
|
|
33762
|
-
}
|
|
33763
|
-
},
|
|
33764
|
-
items: {
|
|
33765
|
-
type: "array",
|
|
33766
|
-
items: {
|
|
33767
|
-
$ref: "#/$defs/record"
|
|
33768
|
-
}
|
|
33769
|
-
}
|
|
33770
|
-
},
|
|
33771
|
-
allOf: [
|
|
33772
|
-
{
|
|
33773
|
-
if: {
|
|
33774
|
-
properties: {
|
|
33775
|
-
schema_version: {
|
|
33776
|
-
const: "aiwg.fortemi.index.export.v1"
|
|
33777
|
-
}
|
|
33778
|
-
}
|
|
33779
|
-
},
|
|
33780
|
-
then: {
|
|
33781
|
-
not: {
|
|
33782
|
-
required: ["compatibility"]
|
|
33783
|
-
},
|
|
33784
|
-
properties: {
|
|
33785
|
-
source: {
|
|
33786
|
-
not: {
|
|
33787
|
-
required: ["graph"]
|
|
33788
|
-
}
|
|
33789
|
-
},
|
|
33790
|
-
items: {
|
|
33791
|
-
items: {
|
|
33792
|
-
allOf: [
|
|
33793
|
-
{
|
|
33794
|
-
properties: {
|
|
33795
|
-
schema_version: {
|
|
33796
|
-
const: "aiwg.fortemi.index.record.v1"
|
|
33797
|
-
}
|
|
33798
|
-
}
|
|
33799
|
-
},
|
|
33800
|
-
{
|
|
33801
|
-
$ref: "#/$defs/v1RecordCompatibility"
|
|
33802
|
-
}
|
|
33803
|
-
]
|
|
33804
|
-
}
|
|
33805
|
-
}
|
|
33806
|
-
}
|
|
33807
|
-
}
|
|
33808
|
-
},
|
|
33809
|
-
{
|
|
33810
|
-
if: {
|
|
33811
|
-
properties: {
|
|
33812
|
-
schema_version: {
|
|
33813
|
-
const: "aiwg.fortemi.index.export.v2"
|
|
33814
|
-
}
|
|
33815
|
-
}
|
|
33816
|
-
},
|
|
33817
|
-
then: {
|
|
33818
|
-
required: ["compatibility"],
|
|
33819
|
-
properties: {
|
|
33820
|
-
source: {
|
|
33821
|
-
required: ["graph"]
|
|
33822
|
-
},
|
|
33823
|
-
items: {
|
|
33824
|
-
items: {
|
|
33825
|
-
properties: {
|
|
33826
|
-
schema_version: {
|
|
33827
|
-
const: "aiwg.fortemi.index.record.v2"
|
|
33828
|
-
}
|
|
33829
|
-
}
|
|
33830
|
-
}
|
|
33831
|
-
}
|
|
33832
|
-
}
|
|
33833
|
-
}
|
|
33834
|
-
}
|
|
33835
|
-
],
|
|
33836
|
-
$defs: {
|
|
33837
|
-
privacy: {
|
|
33838
|
-
enum: ["private", "sanitized", "public"]
|
|
33839
|
-
},
|
|
33840
|
-
recordType: {
|
|
33841
|
-
type: "string",
|
|
33842
|
-
minLength: 1
|
|
33843
|
-
},
|
|
33844
|
-
stringArray: {
|
|
33845
|
-
type: "array",
|
|
33846
|
-
items: {
|
|
33847
|
-
type: "string"
|
|
33848
|
-
}
|
|
33849
|
-
},
|
|
33850
|
-
numberArray: {
|
|
33851
|
-
type: "array",
|
|
33852
|
-
items: {
|
|
33853
|
-
type: "number"
|
|
33854
|
-
}
|
|
33855
|
-
},
|
|
33856
|
-
v1RecordCompatibility: {
|
|
33857
|
-
type: "object",
|
|
33858
|
-
not: {
|
|
33859
|
-
anyOf: [
|
|
33860
|
-
{
|
|
33861
|
-
required: ["name"]
|
|
33862
|
-
},
|
|
33863
|
-
{
|
|
33864
|
-
required: ["summary"]
|
|
33865
|
-
},
|
|
33866
|
-
{
|
|
33867
|
-
required: ["search"]
|
|
33868
|
-
},
|
|
33869
|
-
{
|
|
33870
|
-
required: ["chunks"]
|
|
33871
|
-
},
|
|
33872
|
-
{
|
|
33873
|
-
required: ["embeddings"]
|
|
33874
|
-
},
|
|
33875
|
-
{
|
|
33876
|
-
required: ["compatibility"]
|
|
33877
|
-
},
|
|
33878
|
-
{
|
|
33879
|
-
required: ["skos_concepts"]
|
|
33880
|
-
},
|
|
33881
|
-
{
|
|
33882
|
-
required: ["skos_relations"]
|
|
33883
|
-
},
|
|
33884
|
-
{
|
|
33885
|
-
required: ["provenance_events"]
|
|
33886
|
-
}
|
|
33887
|
-
]
|
|
33888
|
-
},
|
|
33889
|
-
properties: {
|
|
33890
|
-
source: {
|
|
33891
|
-
not: {
|
|
33892
|
-
anyOf: [
|
|
33893
|
-
{
|
|
33894
|
-
required: ["origin"]
|
|
33895
|
-
},
|
|
33896
|
-
{
|
|
33897
|
-
required: ["generated"]
|
|
33898
|
-
},
|
|
33899
|
-
{
|
|
33900
|
-
required: ["checksum"]
|
|
33901
|
-
},
|
|
33902
|
-
{
|
|
33903
|
-
required: ["updated_at"]
|
|
33904
|
-
}
|
|
33905
|
-
]
|
|
33906
|
-
}
|
|
33907
|
-
},
|
|
33908
|
-
relationships: {
|
|
33909
|
-
items: {
|
|
33910
|
-
not: {
|
|
33911
|
-
anyOf: [
|
|
33912
|
-
{
|
|
33913
|
-
required: ["target_path"]
|
|
33914
|
-
},
|
|
33915
|
-
{
|
|
33916
|
-
required: ["direction"]
|
|
33917
|
-
},
|
|
33918
|
-
{
|
|
33919
|
-
required: ["label"]
|
|
33920
|
-
},
|
|
33921
|
-
{
|
|
33922
|
-
required: ["confidence"]
|
|
33923
|
-
},
|
|
33924
|
-
{
|
|
33925
|
-
required: ["privacy"]
|
|
33926
|
-
},
|
|
33927
|
-
{
|
|
33928
|
-
required: ["metadata"]
|
|
33929
|
-
}
|
|
33930
|
-
]
|
|
33931
|
-
}
|
|
33932
|
-
}
|
|
33933
|
-
},
|
|
33934
|
-
privacy: {
|
|
33935
|
-
not: {
|
|
33936
|
-
required: ["locality"]
|
|
33262
|
+
},
|
|
33263
|
+
privacy: {
|
|
33264
|
+
not: {
|
|
33265
|
+
required: ["locality"]
|
|
33937
33266
|
}
|
|
33938
33267
|
}
|
|
33939
33268
|
}
|
|
@@ -34337,98 +33666,1124 @@ var aiwg_fortemi_index_export_schema_default = {
|
|
|
34337
33666
|
}
|
|
34338
33667
|
}
|
|
34339
33668
|
}
|
|
34340
|
-
},
|
|
34341
|
-
|
|
34342
|
-
|
|
34343
|
-
|
|
33669
|
+
},
|
|
33670
|
+
operational_state: {
|
|
33671
|
+
$ref: "#/$defs/operationalState"
|
|
33672
|
+
},
|
|
33673
|
+
state_transfer: {
|
|
33674
|
+
$ref: "#/$defs/stateTransfer"
|
|
33675
|
+
},
|
|
33676
|
+
updated_at: {
|
|
33677
|
+
type: "string",
|
|
33678
|
+
format: "date-time"
|
|
33679
|
+
}
|
|
33680
|
+
}
|
|
33681
|
+
},
|
|
33682
|
+
operationalState: {
|
|
33683
|
+
type: "object",
|
|
33684
|
+
additionalProperties: false,
|
|
33685
|
+
required: ["classification"],
|
|
33686
|
+
properties: {
|
|
33687
|
+
source_repo: { type: "string", minLength: 1 },
|
|
33688
|
+
source_kind: { type: "string", minLength: 1 },
|
|
33689
|
+
source_id: { type: "string", minLength: 1 },
|
|
33690
|
+
observed_state: { type: "string", minLength: 1 },
|
|
33691
|
+
observed_at: { type: "string", format: "date-time" },
|
|
33692
|
+
source_updated_at: { type: "string", format: "date-time" },
|
|
33693
|
+
evidence_url: { type: "string", format: "uri" },
|
|
33694
|
+
evidence_path: { type: "string", minLength: 1 },
|
|
33695
|
+
observer: { type: "string", minLength: 1 },
|
|
33696
|
+
supersedes: { $ref: "#/$defs/stringArray" },
|
|
33697
|
+
contradicts: { $ref: "#/$defs/stringArray" },
|
|
33698
|
+
stale_after: { type: "string", format: "date-time" },
|
|
33699
|
+
classification: {
|
|
33700
|
+
enum: ["fresh", "historical", "superseded", "contradicted", "needs-source"]
|
|
33701
|
+
},
|
|
33702
|
+
confidence: {
|
|
33703
|
+
enum: ["source", "candidate", "reviewed", "rejected"]
|
|
33704
|
+
},
|
|
33705
|
+
current_action_selector: { type: "boolean" }
|
|
33706
|
+
}
|
|
33707
|
+
},
|
|
33708
|
+
stateTransfer: {
|
|
33709
|
+
type: "object",
|
|
33710
|
+
additionalProperties: false,
|
|
33711
|
+
required: ["deleted_at"],
|
|
33712
|
+
properties: {
|
|
33713
|
+
deleted_at: {
|
|
33714
|
+
type: ["string", "null"],
|
|
33715
|
+
format: "date-time"
|
|
33716
|
+
}
|
|
33717
|
+
}
|
|
33718
|
+
}
|
|
33719
|
+
}
|
|
33720
|
+
};
|
|
33721
|
+
|
|
33722
|
+
// src/aiwg-index-schema.ts
|
|
33723
|
+
var PROJECTED_FIELDS = [
|
|
33724
|
+
"schema_version",
|
|
33725
|
+
"id",
|
|
33726
|
+
"type",
|
|
33727
|
+
"title",
|
|
33728
|
+
"text",
|
|
33729
|
+
"facets",
|
|
33730
|
+
"tags",
|
|
33731
|
+
"concepts",
|
|
33732
|
+
"privacy"
|
|
33733
|
+
];
|
|
33734
|
+
var ajvInstance;
|
|
33735
|
+
var exportValidator;
|
|
33736
|
+
var projectedRecordValidator;
|
|
33737
|
+
function getAiwgFortemiIndexExportSchema() {
|
|
33738
|
+
return aiwg_fortemi_index_export_schema_default;
|
|
33739
|
+
}
|
|
33740
|
+
function getAjv() {
|
|
33741
|
+
if (!ajvInstance) {
|
|
33742
|
+
ajvInstance = new Ajv2020({
|
|
33743
|
+
allErrors: true,
|
|
33744
|
+
strict: false,
|
|
33745
|
+
validateFormats: false
|
|
33746
|
+
});
|
|
33747
|
+
ajvInstance.addSchema(aiwg_fortemi_index_export_schema_default);
|
|
33748
|
+
}
|
|
33749
|
+
return ajvInstance;
|
|
33750
|
+
}
|
|
33751
|
+
function formatErrors2(errors) {
|
|
33752
|
+
return (errors ?? []).map((error) => {
|
|
33753
|
+
const path = error.instancePath || "(root)";
|
|
33754
|
+
return `${path} ${error.message ?? "is invalid"}`;
|
|
33755
|
+
});
|
|
33756
|
+
}
|
|
33757
|
+
function getExportValidator() {
|
|
33758
|
+
exportValidator ??= getAjv().getSchema(aiwg_fortemi_index_export_schema_default.$id) ?? getAjv().compile(aiwg_fortemi_index_export_schema_default);
|
|
33759
|
+
return exportValidator;
|
|
33760
|
+
}
|
|
33761
|
+
function getProjectedRecordValidator() {
|
|
33762
|
+
if (!projectedRecordValidator) {
|
|
33763
|
+
const properties = Object.fromEntries(Object.keys(aiwg_fortemi_index_export_schema_default.$defs.record.properties).map((field) => [
|
|
33764
|
+
field,
|
|
33765
|
+
{ $ref: `${aiwg_fortemi_index_export_schema_default.$id}#/$defs/record/properties/${field}` }
|
|
33766
|
+
]));
|
|
33767
|
+
projectedRecordValidator = getAjv().compile({
|
|
33768
|
+
type: "object",
|
|
33769
|
+
required: PROJECTED_FIELDS,
|
|
33770
|
+
properties,
|
|
33771
|
+
additionalProperties: true,
|
|
33772
|
+
allOf: [
|
|
33773
|
+
{
|
|
33774
|
+
if: {
|
|
33775
|
+
properties: {
|
|
33776
|
+
schema_version: { const: "aiwg.fortemi.index.record.v1" }
|
|
33777
|
+
}
|
|
33778
|
+
},
|
|
33779
|
+
then: { $ref: `${aiwg_fortemi_index_export_schema_default.$id}#/$defs/v1RecordCompatibility` }
|
|
33780
|
+
}
|
|
33781
|
+
]
|
|
33782
|
+
});
|
|
33783
|
+
}
|
|
33784
|
+
return projectedRecordValidator;
|
|
33785
|
+
}
|
|
33786
|
+
function validateAiwgFortemiIndexExportSchema(value) {
|
|
33787
|
+
const validate = getExportValidator();
|
|
33788
|
+
const schemaValue = value && typeof value === "object" && Array.isArray(value.items) ? {
|
|
33789
|
+
...value,
|
|
33790
|
+
items: value.items.map((item) => {
|
|
33791
|
+
if (!item || typeof item !== "object" || Array.isArray(item)) return item;
|
|
33792
|
+
const schemaRecord = { ...item };
|
|
33793
|
+
Reflect.deleteProperty(schemaRecord, "binary_sources");
|
|
33794
|
+
return schemaRecord;
|
|
33795
|
+
})
|
|
33796
|
+
} : value;
|
|
33797
|
+
const valid = validate(schemaValue);
|
|
33798
|
+
return { valid, errors: formatErrors2(validate.errors) };
|
|
33799
|
+
}
|
|
33800
|
+
function validateAiwgFortemiProjectedRecordSchema(value) {
|
|
33801
|
+
const validate = getProjectedRecordValidator();
|
|
33802
|
+
const valid = validate(value);
|
|
33803
|
+
return { valid, errors: formatErrors2(validate.errors) };
|
|
33804
|
+
}
|
|
33805
|
+
var encoder4 = new TextEncoder();
|
|
33806
|
+
var UUID_NAMESPACE = "7ab5d1f8-29d2-5e35-9e2f-3a45de171a9e";
|
|
33807
|
+
function uuid(kind, id) {
|
|
33808
|
+
return v5(`${kind}:${id}`, UUID_NAMESPACE);
|
|
33809
|
+
}
|
|
33810
|
+
function own(value, field) {
|
|
33811
|
+
return Object.prototype.hasOwnProperty.call(value, field);
|
|
33812
|
+
}
|
|
33813
|
+
function timestamp(value) {
|
|
33814
|
+
if (!value || Number.isNaN(Date.parse(value))) return void 0;
|
|
33815
|
+
return new Date(value).toISOString();
|
|
33816
|
+
}
|
|
33817
|
+
function addLoss(losses, code, message, details = {}) {
|
|
33818
|
+
losses.push({ code, message, ...details });
|
|
33819
|
+
}
|
|
33820
|
+
function encode(values, encoding) {
|
|
33821
|
+
return encoder4.encode(encoding === "json-array" ? JSON.stringify(values) : values.map((value) => JSON.stringify(value)).join("\n"));
|
|
33822
|
+
}
|
|
33823
|
+
function noteTitle(record, losses) {
|
|
33824
|
+
if (own(record, "title")) return record.title ?? null;
|
|
33825
|
+
const derived = record.search?.title ?? record.search?.name;
|
|
33826
|
+
addLoss(losses, "aiwg-title-not-native", derived ? "Title was derived from the AIWG search projection." : "AIWG supplied no title; the nullable destination field is null.", {
|
|
33827
|
+
component: "notes",
|
|
33828
|
+
record_id: record.id,
|
|
33829
|
+
field_path: "/title",
|
|
33830
|
+
source_state: "absent",
|
|
33831
|
+
destination_capability: "nullable-value",
|
|
33832
|
+
action: derived ? "degrade" : "default"
|
|
33833
|
+
});
|
|
33834
|
+
return derived ?? null;
|
|
33835
|
+
}
|
|
33836
|
+
function noteContent(record, losses) {
|
|
33837
|
+
if (own(record, "text")) return record.text ?? "";
|
|
33838
|
+
if (own(record.search ?? {}, "body")) {
|
|
33839
|
+
addLoss(
|
|
33840
|
+
losses,
|
|
33841
|
+
"aiwg-content-search-projection",
|
|
33842
|
+
"Content was derived from the AIWG search projection.",
|
|
33843
|
+
{
|
|
33844
|
+
component: "notes",
|
|
33845
|
+
record_id: record.id,
|
|
33846
|
+
field_path: "/original_content",
|
|
33847
|
+
source_state: "absent",
|
|
33848
|
+
destination_capability: "required-string",
|
|
33849
|
+
action: "degrade"
|
|
33850
|
+
}
|
|
33851
|
+
);
|
|
33852
|
+
return record.search?.body ?? "";
|
|
33853
|
+
}
|
|
33854
|
+
if (record.chunks?.length) {
|
|
33855
|
+
addLoss(losses, "aiwg-content-chunk-projection", "Content was assembled from AIWG chunks.", {
|
|
33856
|
+
component: "notes",
|
|
33857
|
+
record_id: record.id,
|
|
33858
|
+
field_path: "/original_content",
|
|
33859
|
+
source_state: "absent",
|
|
33860
|
+
destination_capability: "required-string",
|
|
33861
|
+
action: "degrade"
|
|
33862
|
+
});
|
|
33863
|
+
return record.chunks.map((chunk) => {
|
|
33864
|
+
if (own(chunk, "text")) return chunk.text ?? "";
|
|
33865
|
+
if (own(chunk, "body")) return chunk.body ?? "";
|
|
33866
|
+
if (own(chunk, "summary")) return chunk.summary ?? "";
|
|
33867
|
+
return "";
|
|
33868
|
+
}).join("\n\n");
|
|
33869
|
+
}
|
|
33870
|
+
addLoss(
|
|
33871
|
+
losses,
|
|
33872
|
+
"aiwg-content-unavailable",
|
|
33873
|
+
"AIWG supplied no body; the required destination string is empty.",
|
|
33874
|
+
{
|
|
33875
|
+
component: "notes",
|
|
33876
|
+
record_id: record.id,
|
|
33877
|
+
field_path: "/original_content",
|
|
33878
|
+
source_state: "absent",
|
|
33879
|
+
destination_capability: "required-string",
|
|
33880
|
+
action: "default"
|
|
33881
|
+
}
|
|
33882
|
+
);
|
|
33883
|
+
return "";
|
|
33884
|
+
}
|
|
33885
|
+
function createdAt(record, losses) {
|
|
33886
|
+
const source = timestamp(record.source.updated_at);
|
|
33887
|
+
if (source) return source;
|
|
33888
|
+
addLoss(
|
|
33889
|
+
losses,
|
|
33890
|
+
"aiwg-created-at-unavailable",
|
|
33891
|
+
"AIWG has no distinct creation timestamp; updated_at was used.",
|
|
33892
|
+
{
|
|
33893
|
+
component: "notes",
|
|
33894
|
+
record_id: record.id,
|
|
33895
|
+
field_path: "/created_at",
|
|
33896
|
+
source_state: "absent",
|
|
33897
|
+
destination_capability: "required-date-time",
|
|
33898
|
+
action: "default"
|
|
33899
|
+
}
|
|
33900
|
+
);
|
|
33901
|
+
return new Date(record.updated_at).toISOString();
|
|
33902
|
+
}
|
|
33903
|
+
function safeUri(value) {
|
|
33904
|
+
if (!value) return null;
|
|
33905
|
+
try {
|
|
33906
|
+
return new URL(value).toString();
|
|
33907
|
+
} catch {
|
|
33908
|
+
return null;
|
|
33909
|
+
}
|
|
33910
|
+
}
|
|
33911
|
+
function schemeName(concept) {
|
|
33912
|
+
return concept.scheme && concept.scheme.length > 0 ? concept.scheme : "aiwg";
|
|
33913
|
+
}
|
|
33914
|
+
async function convertAiwgIndexToFullV1(index, options = {}) {
|
|
33915
|
+
const losses = [];
|
|
33916
|
+
const exportedAt = timestamp(options.createdAt) ?? new Date(index.generated_at).toISOString();
|
|
33917
|
+
const records = [...index.items].sort((a, b) => a.id.localeCompare(b.id));
|
|
33918
|
+
const noteIds = new Map(records.map((record) => [record.id, uuid("record", record.id)]));
|
|
33919
|
+
const rows = /* @__PURE__ */ new Map();
|
|
33920
|
+
for (const component of Object.keys(FULL_V1_COMPONENT_FILES)) rows.set(component, []);
|
|
33921
|
+
const tags = /* @__PURE__ */ new Set();
|
|
33922
|
+
for (const record of records) {
|
|
33923
|
+
const noteId = noteIds.get(record.id);
|
|
33924
|
+
const content = noteContent(record, losses);
|
|
33925
|
+
const sourceCreatedAt = createdAt(record, losses);
|
|
33926
|
+
const updatedAt = new Date(record.updated_at).toISOString();
|
|
33927
|
+
const recordTags = [...new Set(record.tags)].sort();
|
|
33928
|
+
recordTags.forEach((tag) => tags.add(tag));
|
|
33929
|
+
if (record.binary_sources?.length) {
|
|
33930
|
+
addLoss(
|
|
33931
|
+
losses,
|
|
33932
|
+
"aiwg-attachment-bytes-unavailable",
|
|
33933
|
+
"Attachment references have no bytes, so full-v1 attachment projections were omitted.",
|
|
33934
|
+
{
|
|
33935
|
+
component: "notes",
|
|
33936
|
+
count: record.binary_sources.length,
|
|
33937
|
+
record_id: record.id,
|
|
33938
|
+
field_path: "/attachments",
|
|
33939
|
+
source_state: "value",
|
|
33940
|
+
destination_capability: "mandatory-blob-sidecar",
|
|
33941
|
+
action: "omit"
|
|
33942
|
+
}
|
|
33943
|
+
);
|
|
33944
|
+
}
|
|
33945
|
+
const unmappedRecordFields = [
|
|
33946
|
+
Object.keys(record.facets).length > 0 ? "facets" : null,
|
|
33947
|
+
record.search !== void 0 ? "search" : null,
|
|
33948
|
+
record.chunks !== void 0 ? "chunks" : null,
|
|
33949
|
+
record.compatibility !== void 0 ? "compatibility" : null
|
|
33950
|
+
].filter((field) => field !== null);
|
|
33951
|
+
if (unmappedRecordFields.length > 0) {
|
|
33952
|
+
addLoss(
|
|
33953
|
+
losses,
|
|
33954
|
+
"aiwg-record-fields-unmapped",
|
|
33955
|
+
"AIWG fields without a native full-v1 component were not hidden in note metadata.",
|
|
33956
|
+
{
|
|
33957
|
+
component: "notes",
|
|
33958
|
+
record_id: record.id,
|
|
33959
|
+
source_state: "value",
|
|
33960
|
+
action: "omit",
|
|
33961
|
+
destination_capability: "native-full-v1-components",
|
|
33962
|
+
reason: unmappedRecordFields.join(",")
|
|
33963
|
+
}
|
|
33964
|
+
);
|
|
33965
|
+
}
|
|
33966
|
+
rows.get("notes").push({
|
|
33967
|
+
id: noteId,
|
|
33968
|
+
title: noteTitle(record, losses),
|
|
33969
|
+
original_content: content,
|
|
33970
|
+
revised_content: content,
|
|
33971
|
+
metadata: { aiwg_source: {
|
|
33972
|
+
record_id: record.id,
|
|
33973
|
+
record_type: record.type,
|
|
33974
|
+
repository: index.source.repo,
|
|
33975
|
+
graph: index.source.graph ?? null,
|
|
33976
|
+
source: record.source,
|
|
33977
|
+
privacy: record.privacy
|
|
33978
|
+
} },
|
|
33979
|
+
format: "markdown",
|
|
33980
|
+
source: "aiwg-index",
|
|
33981
|
+
starred: false,
|
|
33982
|
+
archived: false,
|
|
33983
|
+
collection_id: null,
|
|
33984
|
+
created_at: sourceCreatedAt,
|
|
33985
|
+
updated_at: updatedAt,
|
|
33986
|
+
tags: recordTags,
|
|
33987
|
+
attachments: []
|
|
33988
|
+
});
|
|
33989
|
+
const hash = await sha256Hex(encoder4.encode(content));
|
|
33990
|
+
rows.get("note_originals").push({
|
|
33991
|
+
id: uuid("note-original", record.id),
|
|
33992
|
+
note_id: noteId,
|
|
33993
|
+
content,
|
|
33994
|
+
hash: `sha256:${hash}`,
|
|
33995
|
+
user_created_at: sourceCreatedAt,
|
|
33996
|
+
user_last_edited_at: updatedAt,
|
|
33997
|
+
version_number: 1
|
|
33998
|
+
});
|
|
33999
|
+
rows.get("note_revised_current").push({
|
|
34000
|
+
note_id: noteId,
|
|
34001
|
+
content,
|
|
34002
|
+
last_revision_id: null,
|
|
34003
|
+
ai_metadata: null
|
|
34004
|
+
});
|
|
34005
|
+
}
|
|
34006
|
+
rows.set("tags", [...tags].sort().map((name) => ({ name, created_at: exportedAt })));
|
|
34007
|
+
const relationshipInput = [];
|
|
34008
|
+
for (const record of records) {
|
|
34009
|
+
for (const [position, relationship] of record.relationships.entries()) {
|
|
34010
|
+
const target = noteIds.get(relationship.target_id) ?? null;
|
|
34011
|
+
const key = `${record.id}\0${position}\0${relationship.type}\0${relationship.target_id}`;
|
|
34012
|
+
rows.get("links").push({
|
|
34013
|
+
id: uuid("relationship", key),
|
|
34014
|
+
from_note_id: noteIds.get(record.id),
|
|
34015
|
+
to_note_id: target,
|
|
34016
|
+
to_url: target ? null : `aiwg://record/${encodeURIComponent(relationship.target_id)}`,
|
|
34017
|
+
kind: relationship.type,
|
|
34018
|
+
score: own(relationship, "confidence") ? relationship.confidence ?? null : null,
|
|
34019
|
+
created_at: new Date(record.updated_at).toISOString(),
|
|
34020
|
+
metadata: target ? relationship.metadata ?? null : {
|
|
34021
|
+
direction: relationship.direction ?? null,
|
|
34022
|
+
label: relationship.label ?? null,
|
|
34023
|
+
privacy: relationship.privacy ?? null,
|
|
34024
|
+
source_path: relationship.source_path ?? null,
|
|
34025
|
+
target_path: relationship.target_path ?? null,
|
|
34026
|
+
source_metadata: relationship.metadata ?? null
|
|
34027
|
+
}
|
|
34028
|
+
});
|
|
34029
|
+
relationshipInput.push({ source: record.id, position, relationship });
|
|
34030
|
+
if (!target) continue;
|
|
34031
|
+
let weight = relationship.confidence;
|
|
34032
|
+
if (weight === void 0) {
|
|
34033
|
+
weight = 1;
|
|
34034
|
+
addLoss(
|
|
34035
|
+
losses,
|
|
34036
|
+
"aiwg-relationship-weight-unavailable",
|
|
34037
|
+
"Graph weight defaulted because relationship confidence is absent.",
|
|
34038
|
+
{
|
|
34039
|
+
component: "graph_edges",
|
|
34040
|
+
record_id: record.id,
|
|
34041
|
+
field_path: `/relationships/${position}/confidence`,
|
|
34042
|
+
source_state: "absent",
|
|
34043
|
+
destination_capability: "required-number",
|
|
34044
|
+
action: "default"
|
|
34045
|
+
}
|
|
34046
|
+
);
|
|
34047
|
+
}
|
|
34048
|
+
rows.get("graph_edges").push({
|
|
34049
|
+
graph_source_id: "aiwg-relationships",
|
|
34050
|
+
from_note_id: noteIds.get(record.id),
|
|
34051
|
+
to_note_id: target,
|
|
34052
|
+
weight,
|
|
34053
|
+
kind: "manual",
|
|
34054
|
+
rank: position,
|
|
34055
|
+
metadata: {
|
|
34056
|
+
type: relationship.type,
|
|
34057
|
+
direction: relationship.direction ?? null,
|
|
34058
|
+
label: relationship.label ?? null,
|
|
34059
|
+
privacy: relationship.privacy ?? null,
|
|
34060
|
+
source_path: relationship.source_path ?? null,
|
|
34061
|
+
target_path: relationship.target_path ?? null,
|
|
34062
|
+
source_metadata: relationship.metadata ?? null
|
|
34063
|
+
}
|
|
34064
|
+
});
|
|
34065
|
+
}
|
|
34066
|
+
}
|
|
34067
|
+
if (rows.get("graph_edges").length) {
|
|
34068
|
+
rows.get("graph_sources").push({
|
|
34069
|
+
id: "aiwg-relationships",
|
|
34070
|
+
name: "AIWG relationships",
|
|
34071
|
+
kind: "imported",
|
|
34072
|
+
source_table: "manual",
|
|
34073
|
+
embedding_set_id: null,
|
|
34074
|
+
virtual_set_id: null,
|
|
34075
|
+
model: null,
|
|
34076
|
+
dimension: null,
|
|
34077
|
+
truncate_dimension: null,
|
|
34078
|
+
metric: null,
|
|
34079
|
+
algorithm: null,
|
|
34080
|
+
parameters: null,
|
|
34081
|
+
input_hash: `sha256:${await sha256Hex(encoder4.encode(JSON.stringify(relationshipInput)))}`,
|
|
34082
|
+
freshness: { status: "fresh", checked_at: exportedAt },
|
|
34083
|
+
created_at: exportedAt
|
|
34084
|
+
});
|
|
34085
|
+
}
|
|
34086
|
+
const concepts = /* @__PURE__ */ new Map();
|
|
34087
|
+
const conceptMembers = /* @__PURE__ */ new Map();
|
|
34088
|
+
for (const record of records) {
|
|
34089
|
+
for (const id of record.concepts) {
|
|
34090
|
+
conceptMembers.set(id, [...conceptMembers.get(id) ?? [], record]);
|
|
34091
|
+
}
|
|
34092
|
+
for (const concept of record.skos_concepts ?? []) {
|
|
34093
|
+
const existing = concepts.get(concept.id);
|
|
34094
|
+
if (existing && JSON.stringify(existing) !== JSON.stringify(concept)) {
|
|
34095
|
+
addLoss(
|
|
34096
|
+
losses,
|
|
34097
|
+
"aiwg-skos-concept-conflict",
|
|
34098
|
+
"Different metadata was supplied for one concept; the first sorted record won.",
|
|
34099
|
+
{
|
|
34100
|
+
component: "skos_concepts",
|
|
34101
|
+
record_id: concept.id,
|
|
34102
|
+
destination_capability: "single-concept-identity",
|
|
34103
|
+
action: "degrade"
|
|
34104
|
+
}
|
|
34105
|
+
);
|
|
34106
|
+
} else if (!existing) concepts.set(concept.id, concept);
|
|
34107
|
+
}
|
|
34108
|
+
}
|
|
34109
|
+
for (const id of conceptMembers.keys()) {
|
|
34110
|
+
if (!concepts.has(id)) {
|
|
34111
|
+
concepts.set(id, { id, prefLabel: id });
|
|
34112
|
+
addLoss(
|
|
34113
|
+
losses,
|
|
34114
|
+
"aiwg-skos-metadata-unavailable",
|
|
34115
|
+
"A referenced concept had no SKOS metadata; its id is used as the label.",
|
|
34116
|
+
{
|
|
34117
|
+
component: "skos_concepts",
|
|
34118
|
+
record_id: id,
|
|
34119
|
+
field_path: "/prefLabel",
|
|
34120
|
+
source_state: "absent",
|
|
34121
|
+
destination_capability: "required-label",
|
|
34122
|
+
action: "default"
|
|
34123
|
+
}
|
|
34124
|
+
);
|
|
34125
|
+
}
|
|
34126
|
+
}
|
|
34127
|
+
const schemeIds = new Map(
|
|
34128
|
+
[...new Set([...concepts.values()].map(schemeName))].sort().map((name) => [name, uuid("skos-scheme", name)])
|
|
34129
|
+
);
|
|
34130
|
+
for (const [name, id] of schemeIds) {
|
|
34131
|
+
rows.get("skos_schemes").push({
|
|
34132
|
+
id,
|
|
34133
|
+
uri: null,
|
|
34134
|
+
notation: name,
|
|
34135
|
+
title: name,
|
|
34136
|
+
description: null,
|
|
34137
|
+
creator: "aiwg-index",
|
|
34138
|
+
publisher: null,
|
|
34139
|
+
rights: null,
|
|
34140
|
+
version: null,
|
|
34141
|
+
is_active: true,
|
|
34142
|
+
is_system: false,
|
|
34143
|
+
created_at: exportedAt,
|
|
34144
|
+
updated_at: exportedAt,
|
|
34145
|
+
issued_at: null,
|
|
34146
|
+
modified_at: null,
|
|
34147
|
+
embedding: null,
|
|
34148
|
+
embedding_model: null,
|
|
34149
|
+
embedded_at: null
|
|
34150
|
+
});
|
|
34151
|
+
}
|
|
34152
|
+
const conceptIds = new Map([...concepts.keys()].sort().map((id) => [id, uuid("skos-concept", id)]));
|
|
34153
|
+
for (const [sourceId, concept] of [...concepts].sort(([a], [b]) => a.localeCompare(b))) {
|
|
34154
|
+
const id = conceptIds.get(sourceId);
|
|
34155
|
+
const schemeId = schemeIds.get(schemeName(concept));
|
|
34156
|
+
const memberCount = conceptMembers.get(sourceId)?.length ?? 0;
|
|
34157
|
+
if (concept.uri && !safeUri(concept.uri)) addLoss(
|
|
34158
|
+
losses,
|
|
34159
|
+
"aiwg-skos-uri-invalid",
|
|
34160
|
+
"AIWG concept URI is not a valid authority URI and was mapped to null.",
|
|
34161
|
+
{
|
|
34162
|
+
component: "skos_concepts",
|
|
34163
|
+
record_id: sourceId,
|
|
34164
|
+
field_path: "/uri",
|
|
34165
|
+
source_state: "value",
|
|
34166
|
+
destination_capability: "uri",
|
|
34167
|
+
action: "omit"
|
|
34168
|
+
}
|
|
34169
|
+
);
|
|
34170
|
+
const unmappedConceptMetadata = Object.keys(concept.metadata ?? {}).filter((key) => key !== "domain");
|
|
34171
|
+
if (unmappedConceptMetadata.length > 0) addLoss(
|
|
34172
|
+
losses,
|
|
34173
|
+
"aiwg-skos-metadata-unmapped",
|
|
34174
|
+
"AIWG concept metadata without a native full-v1 field was omitted.",
|
|
34175
|
+
{
|
|
34176
|
+
component: "skos_concepts",
|
|
34177
|
+
record_id: sourceId,
|
|
34178
|
+
source_state: "value",
|
|
34179
|
+
action: "omit",
|
|
34180
|
+
destination_capability: "native-skos-fields",
|
|
34181
|
+
reason: unmappedConceptMetadata.join(",")
|
|
34182
|
+
}
|
|
34183
|
+
);
|
|
34184
|
+
rows.get("skos_concepts").push({
|
|
34185
|
+
id,
|
|
34186
|
+
primary_scheme_id: schemeId,
|
|
34187
|
+
uri: safeUri(concept.uri),
|
|
34188
|
+
notation: concept.notation ?? sourceId,
|
|
34189
|
+
facet_type: null,
|
|
34190
|
+
facet_source: "aiwg-index",
|
|
34191
|
+
facet_domain: typeof concept.metadata?.domain === "string" ? concept.metadata.domain : null,
|
|
34192
|
+
facet_scope: null,
|
|
34193
|
+
status: "candidate",
|
|
34194
|
+
promoted_at: null,
|
|
34195
|
+
deprecated_at: null,
|
|
34196
|
+
deprecation_reason: null,
|
|
34197
|
+
replaced_by_id: null,
|
|
34198
|
+
note_count: memberCount,
|
|
34199
|
+
first_used_at: memberCount ? exportedAt : null,
|
|
34200
|
+
last_used_at: memberCount ? exportedAt : null,
|
|
34201
|
+
depth: 0,
|
|
34202
|
+
broader_count: 0,
|
|
34203
|
+
narrower_count: 0,
|
|
34204
|
+
related_count: 0,
|
|
34205
|
+
antipatterns: null,
|
|
34206
|
+
antipattern_checked_at: null,
|
|
34207
|
+
created_at: exportedAt,
|
|
34208
|
+
updated_at: exportedAt,
|
|
34209
|
+
embedding: null,
|
|
34210
|
+
embedding_model: null,
|
|
34211
|
+
embedded_at: null
|
|
34212
|
+
});
|
|
34213
|
+
rows.get("skos_scheme_memberships").push({
|
|
34214
|
+
concept_id: id,
|
|
34215
|
+
scheme_id: schemeId,
|
|
34216
|
+
is_top_concept: true,
|
|
34217
|
+
added_at: exportedAt
|
|
34218
|
+
});
|
|
34219
|
+
rows.get("skos_labels").push({
|
|
34220
|
+
id: uuid("skos-label", `${sourceId}:pref:${concept.prefLabel}`),
|
|
34221
|
+
concept_id: id,
|
|
34222
|
+
label_type: "pref_label",
|
|
34223
|
+
value: concept.prefLabel,
|
|
34224
|
+
language: "und",
|
|
34225
|
+
created_at: exportedAt
|
|
34226
|
+
});
|
|
34227
|
+
for (const label of concept.altLabels ?? []) rows.get("skos_labels").push({
|
|
34228
|
+
id: uuid("skos-label", `${sourceId}:alt:${label}`),
|
|
34229
|
+
concept_id: id,
|
|
34230
|
+
label_type: "alt_label",
|
|
34231
|
+
value: label,
|
|
34232
|
+
language: "und",
|
|
34233
|
+
created_at: exportedAt
|
|
34234
|
+
});
|
|
34235
|
+
if (concept.definition !== void 0) rows.get("skos_notes").push({
|
|
34236
|
+
id: uuid("skos-note", `${sourceId}:definition`),
|
|
34237
|
+
concept_id: id,
|
|
34238
|
+
note_type: "definition",
|
|
34239
|
+
value: concept.definition,
|
|
34240
|
+
language: "und",
|
|
34241
|
+
author: null,
|
|
34242
|
+
source: concept.uri ?? null,
|
|
34243
|
+
created_at: exportedAt,
|
|
34244
|
+
updated_at: exportedAt
|
|
34245
|
+
});
|
|
34246
|
+
}
|
|
34247
|
+
for (const record of records) {
|
|
34248
|
+
for (const [position, sourceId] of record.concepts.entries()) {
|
|
34249
|
+
const conceptId = conceptIds.get(sourceId);
|
|
34250
|
+
if (conceptId) rows.get("note_skos_tags").push({
|
|
34251
|
+
note_id: noteIds.get(record.id),
|
|
34252
|
+
concept_id: conceptId,
|
|
34253
|
+
source: "aiwg-index",
|
|
34254
|
+
confidence: null,
|
|
34255
|
+
relevance_score: null,
|
|
34256
|
+
is_primary: position === 0,
|
|
34257
|
+
created_at: new Date(record.updated_at).toISOString(),
|
|
34258
|
+
created_by: "aiwg-index"
|
|
34259
|
+
});
|
|
34260
|
+
}
|
|
34261
|
+
for (const relation of record.skos_relations ?? []) {
|
|
34262
|
+
const subject = conceptIds.get(relation.source_id);
|
|
34263
|
+
const object = conceptIds.get(relation.target_id);
|
|
34264
|
+
if (!subject || !object || !["broader", "narrower", "related"].includes(relation.type)) {
|
|
34265
|
+
addLoss(
|
|
34266
|
+
losses,
|
|
34267
|
+
"aiwg-skos-relation-unrepresentable",
|
|
34268
|
+
"SKOS relation could not be mapped to the native full-v1 relation enum.",
|
|
34269
|
+
{
|
|
34270
|
+
component: "skos_relations",
|
|
34271
|
+
record_id: record.id,
|
|
34272
|
+
destination_capability: "broader-narrower-related",
|
|
34273
|
+
action: "omit"
|
|
34274
|
+
}
|
|
34275
|
+
);
|
|
34276
|
+
continue;
|
|
34277
|
+
}
|
|
34278
|
+
const unmappedRelationFields = [
|
|
34279
|
+
relation.source_path !== void 0 ? "source_path" : null,
|
|
34280
|
+
relation.metadata !== void 0 ? "metadata" : null
|
|
34281
|
+
].filter((field) => field !== null);
|
|
34282
|
+
if (unmappedRelationFields.length > 0) {
|
|
34283
|
+
addLoss(
|
|
34284
|
+
losses,
|
|
34285
|
+
"aiwg-skos-relation-fields-unmapped",
|
|
34286
|
+
"AIWG SKOS relation fields have no native full-v1 destination fields.",
|
|
34287
|
+
{
|
|
34288
|
+
component: "skos_relations",
|
|
34289
|
+
record_id: record.id,
|
|
34290
|
+
source_state: "value",
|
|
34291
|
+
destination_capability: "native-skos-relation-fields",
|
|
34292
|
+
action: "omit",
|
|
34293
|
+
reason: unmappedRelationFields.join(",")
|
|
34294
|
+
}
|
|
34295
|
+
);
|
|
34296
|
+
continue;
|
|
34297
|
+
}
|
|
34298
|
+
rows.get("skos_relations").push({
|
|
34299
|
+
id: uuid("skos-relation", `${relation.source_id}:${relation.type}:${relation.target_id}`),
|
|
34300
|
+
subject_id: subject,
|
|
34301
|
+
object_id: object,
|
|
34302
|
+
relation_type: relation.type,
|
|
34303
|
+
inference_score: null,
|
|
34304
|
+
is_inferred: false,
|
|
34305
|
+
is_validated: false,
|
|
34306
|
+
created_at: new Date(record.updated_at).toISOString(),
|
|
34307
|
+
created_by: "aiwg-index"
|
|
34308
|
+
});
|
|
34309
|
+
}
|
|
34310
|
+
}
|
|
34311
|
+
for (const record of records) {
|
|
34312
|
+
for (const [position, event] of (record.provenance_events ?? []).entries()) {
|
|
34313
|
+
const start = timestamp(event.started_at);
|
|
34314
|
+
if (!start) addLoss(
|
|
34315
|
+
losses,
|
|
34316
|
+
"aiwg-provenance-start-unavailable",
|
|
34317
|
+
"Activity start defaulted to record updated_at.",
|
|
34318
|
+
{
|
|
34319
|
+
component: "provenance_activities",
|
|
34320
|
+
record_id: event.id ?? `${record.id}:${position}`,
|
|
34321
|
+
field_path: "/started_at",
|
|
34322
|
+
source_state: "absent",
|
|
34323
|
+
action: "default"
|
|
34324
|
+
}
|
|
34325
|
+
);
|
|
34326
|
+
rows.get("provenance_activities").push({
|
|
34327
|
+
id: uuid("provenance-event", event.id ?? `${record.id}:${position}`),
|
|
34328
|
+
note_id: noteIds.get(record.id),
|
|
34329
|
+
revision_id: null,
|
|
34330
|
+
activity_type: event.activity,
|
|
34331
|
+
model_name: event.agent ?? null,
|
|
34332
|
+
started_at: start ?? new Date(record.updated_at).toISOString(),
|
|
34333
|
+
ended_at: timestamp(event.ended_at) ?? null,
|
|
34334
|
+
metadata: {
|
|
34335
|
+
source: event.source ?? null,
|
|
34336
|
+
path: event.path ?? null,
|
|
34337
|
+
confidence: event.confidence ?? null,
|
|
34338
|
+
privacy: event.privacy ?? null,
|
|
34339
|
+
attributes: event.attributes ?? null
|
|
34340
|
+
}
|
|
34341
|
+
});
|
|
34342
|
+
}
|
|
34343
|
+
for (const [position, provenance] of record.provenance.entries()) {
|
|
34344
|
+
rows.get("provenance_activities").push({
|
|
34345
|
+
id: uuid("provenance-field", `${record.id}:${position}:${provenance.field}`),
|
|
34346
|
+
note_id: noteIds.get(record.id),
|
|
34347
|
+
revision_id: null,
|
|
34348
|
+
activity_type: `source:${provenance.field}`,
|
|
34349
|
+
model_name: provenance.source,
|
|
34350
|
+
started_at: new Date(record.updated_at).toISOString(),
|
|
34351
|
+
ended_at: null,
|
|
34352
|
+
metadata: { path: provenance.path, confidence: provenance.confidence, privacy: provenance.privacy }
|
|
34353
|
+
});
|
|
34354
|
+
}
|
|
34355
|
+
}
|
|
34356
|
+
const embeddingGroups = /* @__PURE__ */ new Map();
|
|
34357
|
+
for (const record of records) {
|
|
34358
|
+
for (const [position, embedding] of (record.embeddings ?? []).entries()) {
|
|
34359
|
+
const vector = embedding.embedding ?? embedding.vector;
|
|
34360
|
+
if (!vector || vector.length !== 768) {
|
|
34361
|
+
addLoss(
|
|
34362
|
+
losses,
|
|
34363
|
+
"aiwg-embedding-dimension-unrepresentable",
|
|
34364
|
+
"Only 768-dimensional vectors satisfy the full-v1 authority.",
|
|
34365
|
+
{
|
|
34366
|
+
component: "embeddings",
|
|
34367
|
+
record_id: embedding.id ?? `${record.id}:${position}`,
|
|
34368
|
+
field_path: "/vector",
|
|
34369
|
+
source_state: vector ? "value" : "absent",
|
|
34370
|
+
destination_capability: "768-dimensional-vector",
|
|
34371
|
+
action: "omit"
|
|
34372
|
+
}
|
|
34373
|
+
);
|
|
34374
|
+
continue;
|
|
34375
|
+
}
|
|
34376
|
+
const model = embedding.model ?? "aiwg-unknown";
|
|
34377
|
+
if (!embedding.model) addLoss(
|
|
34378
|
+
losses,
|
|
34379
|
+
"aiwg-embedding-model-unavailable",
|
|
34380
|
+
"Embedding model defaulted because AIWG did not supply one.",
|
|
34381
|
+
{
|
|
34382
|
+
component: "embeddings",
|
|
34383
|
+
record_id: embedding.id ?? `${record.id}:${position}`,
|
|
34384
|
+
field_path: "/model",
|
|
34385
|
+
source_state: "absent",
|
|
34386
|
+
action: "default"
|
|
34344
34387
|
}
|
|
34388
|
+
);
|
|
34389
|
+
let group = embeddingGroups.get(model);
|
|
34390
|
+
if (!group) {
|
|
34391
|
+
group = { config: uuid("embedding-config", model), set: uuid("embedding-set", model), count: 0 };
|
|
34392
|
+
embeddingGroups.set(model, group);
|
|
34345
34393
|
}
|
|
34394
|
+
group.count += 1;
|
|
34395
|
+
const unmappedEmbeddingFields = [
|
|
34396
|
+
embedding.granularity !== void 0 ? "granularity" : null,
|
|
34397
|
+
embedding.source_path !== void 0 ? "source_path" : null,
|
|
34398
|
+
embedding.metadata !== void 0 ? "metadata" : null,
|
|
34399
|
+
embedding.input_hash !== void 0 && !/^[0-9a-f]{64}$/.test(embedding.input_hash) ? "input_hash" : null
|
|
34400
|
+
].filter((field) => field !== null);
|
|
34401
|
+
if (unmappedEmbeddingFields.length > 0) addLoss(
|
|
34402
|
+
losses,
|
|
34403
|
+
"aiwg-embedding-fields-unmapped",
|
|
34404
|
+
"AIWG embedding fields without a native full-v1 field were omitted.",
|
|
34405
|
+
{
|
|
34406
|
+
component: "embeddings",
|
|
34407
|
+
record_id: embedding.id ?? `${record.id}:${position}`,
|
|
34408
|
+
source_state: "value",
|
|
34409
|
+
action: "omit",
|
|
34410
|
+
destination_capability: "native-embedding-fields",
|
|
34411
|
+
reason: unmappedEmbeddingFields.join(",")
|
|
34412
|
+
}
|
|
34413
|
+
);
|
|
34414
|
+
rows.get("embeddings").push({
|
|
34415
|
+
id: uuid("embedding", embedding.id ?? `${record.id}:${position}`),
|
|
34416
|
+
note_id: noteIds.get(record.id),
|
|
34417
|
+
embedding_set_id: group.set,
|
|
34418
|
+
chunk_index: position,
|
|
34419
|
+
text: record.text ?? "",
|
|
34420
|
+
vector,
|
|
34421
|
+
model,
|
|
34422
|
+
contract_fingerprint: /^[0-9a-f]{64}$/.test(embedding.input_hash ?? "") ? embedding.input_hash : null,
|
|
34423
|
+
created_at: new Date(record.updated_at).toISOString()
|
|
34424
|
+
});
|
|
34346
34425
|
}
|
|
34347
34426
|
}
|
|
34348
|
-
|
|
34349
|
-
|
|
34350
|
-
|
|
34351
|
-
|
|
34352
|
-
|
|
34353
|
-
|
|
34354
|
-
|
|
34355
|
-
|
|
34356
|
-
|
|
34357
|
-
|
|
34358
|
-
|
|
34359
|
-
|
|
34360
|
-
|
|
34361
|
-
|
|
34362
|
-
|
|
34363
|
-
|
|
34364
|
-
|
|
34365
|
-
|
|
34366
|
-
|
|
34367
|
-
|
|
34368
|
-
|
|
34369
|
-
|
|
34370
|
-
|
|
34371
|
-
|
|
34372
|
-
|
|
34373
|
-
|
|
34427
|
+
for (const [model, group] of [...embeddingGroups].sort(([a], [b]) => a.localeCompare(b))) {
|
|
34428
|
+
rows.get("embedding_configs").push({
|
|
34429
|
+
id: group.config,
|
|
34430
|
+
name: model,
|
|
34431
|
+
description: null,
|
|
34432
|
+
model,
|
|
34433
|
+
dimension: 768,
|
|
34434
|
+
chunk_size: 1,
|
|
34435
|
+
chunk_overlap: 0,
|
|
34436
|
+
hnsw_m: null,
|
|
34437
|
+
hnsw_ef_construction: null,
|
|
34438
|
+
ivfflat_lists: null,
|
|
34439
|
+
is_default: null,
|
|
34440
|
+
supports_mrl: null,
|
|
34441
|
+
matryoshka_dims: null,
|
|
34442
|
+
default_truncate_dim: null,
|
|
34443
|
+
provider: null,
|
|
34444
|
+
provider_config: null,
|
|
34445
|
+
content_types: null,
|
|
34446
|
+
strengths: null,
|
|
34447
|
+
limitations: null,
|
|
34448
|
+
recommended_for: null,
|
|
34449
|
+
benchmark_scores: null,
|
|
34450
|
+
is_available: null,
|
|
34451
|
+
document_composition: {},
|
|
34452
|
+
created_at: exportedAt,
|
|
34453
|
+
updated_at: exportedAt
|
|
34454
|
+
});
|
|
34455
|
+
rows.get("embedding_sets").push({
|
|
34456
|
+
id: group.set,
|
|
34457
|
+
name: model,
|
|
34458
|
+
slug: uuid("embedding-slug", model),
|
|
34459
|
+
description: null,
|
|
34460
|
+
purpose: null,
|
|
34461
|
+
usage_hints: null,
|
|
34462
|
+
keywords: null,
|
|
34463
|
+
set_type: "full",
|
|
34464
|
+
mode: "manual",
|
|
34465
|
+
criteria: null,
|
|
34466
|
+
embedding_config_id: group.config,
|
|
34467
|
+
truncate_dim: null,
|
|
34468
|
+
auto_embed_rules: null,
|
|
34469
|
+
index_status: "ready",
|
|
34470
|
+
index_type: null,
|
|
34471
|
+
last_indexed_at: exportedAt,
|
|
34472
|
+
document_count: null,
|
|
34473
|
+
embedding_count: group.count,
|
|
34474
|
+
embeddings_current: true,
|
|
34475
|
+
index_size_bytes: null,
|
|
34476
|
+
is_system: false,
|
|
34477
|
+
is_active: true,
|
|
34478
|
+
auto_refresh: false,
|
|
34479
|
+
refresh_interval: null,
|
|
34480
|
+
last_refresh_at: null,
|
|
34481
|
+
agent_metadata: null,
|
|
34482
|
+
created_at: exportedAt,
|
|
34483
|
+
updated_at: exportedAt,
|
|
34484
|
+
created_by: "aiwg-index"
|
|
34485
|
+
});
|
|
34486
|
+
const noteIdSet = new Set(rows.get("embeddings").filter((item) => item.embedding_set_id === group.set).map((item) => item.note_id));
|
|
34487
|
+
for (const noteId of [...noteIdSet].sort()) rows.get("embedding_set_members").push({
|
|
34488
|
+
embedding_set_id: group.set,
|
|
34489
|
+
note_id: noteId,
|
|
34490
|
+
membership_type: "materialized",
|
|
34491
|
+
added_at: exportedAt,
|
|
34492
|
+
added_by: "aiwg-index"
|
|
34374
34493
|
});
|
|
34375
|
-
ajvInstance.addSchema(aiwg_fortemi_index_export_schema_default);
|
|
34376
34494
|
}
|
|
34377
|
-
|
|
34495
|
+
if (losses.length > 0) {
|
|
34496
|
+
return {
|
|
34497
|
+
success: false,
|
|
34498
|
+
archive: null,
|
|
34499
|
+
profile: "full-v1",
|
|
34500
|
+
schema_version: "2.0.0",
|
|
34501
|
+
lossless: false,
|
|
34502
|
+
losses,
|
|
34503
|
+
receipt: {
|
|
34504
|
+
schema_version: "fortemi.aiwg-full-v1-conversion-receipt.v1",
|
|
34505
|
+
source_schema_version: "aiwg.fortemi.index.export.v2",
|
|
34506
|
+
authority_repository: knowledge_shard_v2_schema_receipt_default.source.repository,
|
|
34507
|
+
authority_commit: knowledge_shard_v2_schema_receipt_default.source.commit,
|
|
34508
|
+
authority_contract_sha256: knowledge_shard_v2_schema_receipt_default.source.contractSha256,
|
|
34509
|
+
authority_schema_bundle_sha256: knowledge_shard_v2_schema_receipt_default.schemaBundle.sha256,
|
|
34510
|
+
manifest_sha256: null,
|
|
34511
|
+
component_checksums: {},
|
|
34512
|
+
contract_valid: false,
|
|
34513
|
+
signed: false
|
|
34514
|
+
}
|
|
34515
|
+
};
|
|
34516
|
+
}
|
|
34517
|
+
const files = /* @__PURE__ */ new Map();
|
|
34518
|
+
const components = Object.keys(FULL_V1_COMPONENT_FILES);
|
|
34519
|
+
const counts = { community_sets: 0 };
|
|
34520
|
+
for (const component of components) {
|
|
34521
|
+
const spec = FULL_V1_COMPONENT_FILES[component];
|
|
34522
|
+
const componentRows = rows.get(component) ?? [];
|
|
34523
|
+
files.set(spec.file, encode(componentRows, spec.encoding));
|
|
34524
|
+
counts[component] = componentRows.length;
|
|
34525
|
+
}
|
|
34526
|
+
const checksums = {};
|
|
34527
|
+
for (const [file, bytes] of files) checksums[file] = await sha256Hex(bytes);
|
|
34528
|
+
const manifest = {
|
|
34529
|
+
version: "2.0.0",
|
|
34530
|
+
profile: "full-v1",
|
|
34531
|
+
producer: { name: "fortemi-core-aiwg-index", version: options.matricVersion ?? "fortemi-core" },
|
|
34532
|
+
format: SHARD_FORMAT,
|
|
34533
|
+
created_at: exportedAt,
|
|
34534
|
+
components,
|
|
34535
|
+
counts,
|
|
34536
|
+
checksums,
|
|
34537
|
+
min_reader_version: "2.0.0"
|
|
34538
|
+
};
|
|
34539
|
+
const manifestBytes = encoder4.encode(JSON.stringify(manifest, null, 2));
|
|
34540
|
+
files.set("manifest.json", manifestBytes);
|
|
34541
|
+
const validation = await validateFullV1ShardArchive(files);
|
|
34542
|
+
if (!validation.valid) {
|
|
34543
|
+
throw new Error(`Generated AIWG full-v1 shard failed authority validation: ${validation.errors.join("; ")}`);
|
|
34544
|
+
}
|
|
34545
|
+
return {
|
|
34546
|
+
success: true,
|
|
34547
|
+
archive: packTarGz(files),
|
|
34548
|
+
profile: "full-v1",
|
|
34549
|
+
schema_version: "2.0.0",
|
|
34550
|
+
lossless: losses.length === 0,
|
|
34551
|
+
losses,
|
|
34552
|
+
receipt: {
|
|
34553
|
+
schema_version: "fortemi.aiwg-full-v1-conversion-receipt.v1",
|
|
34554
|
+
source_schema_version: "aiwg.fortemi.index.export.v2",
|
|
34555
|
+
authority_repository: knowledge_shard_v2_schema_receipt_default.source.repository,
|
|
34556
|
+
authority_commit: knowledge_shard_v2_schema_receipt_default.source.commit,
|
|
34557
|
+
authority_contract_sha256: knowledge_shard_v2_schema_receipt_default.source.contractSha256,
|
|
34558
|
+
authority_schema_bundle_sha256: knowledge_shard_v2_schema_receipt_default.schemaBundle.sha256,
|
|
34559
|
+
manifest_sha256: await sha256Hex(manifestBytes),
|
|
34560
|
+
component_checksums: checksums,
|
|
34561
|
+
contract_valid: true,
|
|
34562
|
+
signed: false
|
|
34563
|
+
}
|
|
34564
|
+
};
|
|
34378
34565
|
}
|
|
34379
|
-
|
|
34380
|
-
|
|
34381
|
-
|
|
34382
|
-
|
|
34383
|
-
|
|
34566
|
+
|
|
34567
|
+
// src/aiwg-index-shard.ts
|
|
34568
|
+
var AIWG_SHARD_UUID_NAMESPACE = "7ab5d1f8-29d2-5e35-9e2f-3a45de171a9e";
|
|
34569
|
+
var shardEncoder = new TextEncoder();
|
|
34570
|
+
var shardDecoder = new TextDecoder();
|
|
34571
|
+
function aiwgShardUuid(kind, id) {
|
|
34572
|
+
return v5(`${kind}:${id}`, AIWG_SHARD_UUID_NAMESPACE);
|
|
34384
34573
|
}
|
|
34385
|
-
function
|
|
34386
|
-
|
|
34387
|
-
return
|
|
34574
|
+
function aiwgShardTimestamp(value, fallback) {
|
|
34575
|
+
if (!value || Number.isNaN(Date.parse(value))) return fallback;
|
|
34576
|
+
return new Date(value).toISOString();
|
|
34388
34577
|
}
|
|
34389
|
-
function
|
|
34390
|
-
|
|
34391
|
-
|
|
34392
|
-
|
|
34393
|
-
|
|
34394
|
-
|
|
34395
|
-
|
|
34396
|
-
|
|
34397
|
-
|
|
34398
|
-
|
|
34399
|
-
|
|
34400
|
-
|
|
34401
|
-
|
|
34402
|
-
|
|
34403
|
-
|
|
34404
|
-
|
|
34405
|
-
|
|
34406
|
-
|
|
34407
|
-
|
|
34578
|
+
function aiwgRecordTitle(record) {
|
|
34579
|
+
return record.title ?? record.search?.title ?? record.search?.name ?? record.id;
|
|
34580
|
+
}
|
|
34581
|
+
function aiwgRecordContent(record) {
|
|
34582
|
+
if (record.text) return record.text;
|
|
34583
|
+
if (record.search?.body) return record.search.body;
|
|
34584
|
+
const chunks = record.chunks?.map((chunk) => chunk.text ?? chunk.body ?? chunk.summary ?? "").filter(Boolean);
|
|
34585
|
+
if (chunks?.length) return chunks.join("\n\n");
|
|
34586
|
+
return record.search?.summary ?? "";
|
|
34587
|
+
}
|
|
34588
|
+
function aiwgRecordCollectionPaths(record) {
|
|
34589
|
+
const segments = record.source.repo_relative_path.split("/").filter((segment) => segment.length > 0 && segment !== ".");
|
|
34590
|
+
const directorySegments = segments.slice(0, -1);
|
|
34591
|
+
return directorySegments.map((_, index) => directorySegments.slice(0, index + 1).join("/"));
|
|
34592
|
+
}
|
|
34593
|
+
function aiwgShardMetadata(index, record) {
|
|
34594
|
+
return {
|
|
34595
|
+
aiwg_fortemi_index: {
|
|
34596
|
+
envelope: {
|
|
34597
|
+
schema_version: index.schema_version,
|
|
34598
|
+
generated_at: index.generated_at,
|
|
34599
|
+
source: index.source,
|
|
34600
|
+
...index.compatibility ? { compatibility: index.compatibility } : {}
|
|
34601
|
+
},
|
|
34602
|
+
record
|
|
34603
|
+
}
|
|
34604
|
+
};
|
|
34605
|
+
}
|
|
34606
|
+
function encodeJsonLines(values) {
|
|
34607
|
+
return shardEncoder.encode(values.map((value) => JSON.stringify(value)).join("\n"));
|
|
34608
|
+
}
|
|
34609
|
+
function validateAiwgIndexForShard(index) {
|
|
34610
|
+
const contractValidation = validateAiwgFortemiIndexExportSchema(index);
|
|
34611
|
+
const semanticValidation = validateAiwgFortemiIndexExport(index);
|
|
34612
|
+
const errors = [...contractValidation.errors, ...semanticValidation.errors];
|
|
34613
|
+
if (errors.length > 0) {
|
|
34614
|
+
throw new Error(`Invalid AIWG Fortemi index export:
|
|
34615
|
+
${errors.join("\n")}`);
|
|
34616
|
+
}
|
|
34617
|
+
}
|
|
34618
|
+
async function aiwgFortemiIndexToKnowledgeShard(index, options = {}) {
|
|
34619
|
+
if (options.includeNativeRichComponents) {
|
|
34620
|
+
throw new Error(
|
|
34621
|
+
"Native AIWG full-v1 conversion requires aiwgFortemiIndexToKnowledgeShardWithReport so semantic losses cannot be discarded"
|
|
34622
|
+
);
|
|
34623
|
+
}
|
|
34624
|
+
validateAiwgIndexForShard(index);
|
|
34625
|
+
if (index.schema_version !== "aiwg.fortemi.index.export.v2") {
|
|
34626
|
+
throw new Error("Knowledge Shard conversion requires aiwg.fortemi.index.export.v2");
|
|
34627
|
+
}
|
|
34628
|
+
const createdAt2 = aiwgShardTimestamp(options.createdAt ?? index.generated_at, (/* @__PURE__ */ new Date()).toISOString());
|
|
34629
|
+
const noteIds = new Map(index.items.map((record) => [record.id, aiwgShardUuid("record", record.id)]));
|
|
34630
|
+
const collectionPaths = [...new Set(index.items.flatMap(aiwgRecordCollectionPaths))].sort();
|
|
34631
|
+
const collectionIds = new Map(
|
|
34632
|
+
collectionPaths.map((path) => [path, aiwgShardUuid("collection", path)])
|
|
34633
|
+
);
|
|
34634
|
+
const collectionNoteCounts = /* @__PURE__ */ new Map();
|
|
34635
|
+
for (const record of index.items) {
|
|
34636
|
+
const collectionPath = aiwgRecordCollectionPaths(record).at(-1);
|
|
34637
|
+
if (collectionPath) {
|
|
34638
|
+
collectionNoteCounts.set(collectionPath, (collectionNoteCounts.get(collectionPath) ?? 0) + 1);
|
|
34639
|
+
}
|
|
34640
|
+
}
|
|
34641
|
+
const collections = collectionPaths.map((path) => {
|
|
34642
|
+
const segments = path.split("/");
|
|
34643
|
+
const parentPath = segments.slice(0, -1).join("/");
|
|
34644
|
+
return {
|
|
34645
|
+
id: collectionIds.get(path),
|
|
34646
|
+
name: segments.at(-1),
|
|
34647
|
+
description: null,
|
|
34648
|
+
parent_id: parentPath ? collectionIds.get(parentPath) : null,
|
|
34649
|
+
created_at: createdAt2,
|
|
34650
|
+
note_count: collectionNoteCounts.get(path) ?? 0
|
|
34651
|
+
};
|
|
34652
|
+
});
|
|
34653
|
+
const notes = index.items.map((record) => {
|
|
34654
|
+
const content = aiwgRecordContent(record);
|
|
34655
|
+
const recordCollectionPaths = aiwgRecordCollectionPaths(record);
|
|
34656
|
+
const collectionPath = recordCollectionPaths.at(-1);
|
|
34657
|
+
return {
|
|
34658
|
+
id: noteIds.get(record.id),
|
|
34659
|
+
title: aiwgRecordTitle(record),
|
|
34660
|
+
original_content: content,
|
|
34661
|
+
revised_content: content,
|
|
34662
|
+
metadata: aiwgShardMetadata(index, record),
|
|
34663
|
+
collection_id: collectionPath ? collectionIds.get(collectionPath) : null,
|
|
34664
|
+
attachments: [],
|
|
34665
|
+
format: "markdown",
|
|
34666
|
+
source: "aiwg-index",
|
|
34667
|
+
starred: false,
|
|
34668
|
+
archived: false,
|
|
34669
|
+
tags: [...new Set(record.tags)].sort(),
|
|
34670
|
+
created_at: aiwgShardTimestamp(record.source.updated_at ?? record.updated_at, createdAt2),
|
|
34671
|
+
updated_at: aiwgShardTimestamp(record.updated_at, createdAt2),
|
|
34672
|
+
deleted_at: record.state_transfer?.deleted_at ?? null
|
|
34673
|
+
};
|
|
34674
|
+
});
|
|
34675
|
+
const links = [];
|
|
34676
|
+
for (const record of index.items) {
|
|
34677
|
+
for (const [position, relationship] of record.relationships.entries()) {
|
|
34678
|
+
const targetNoteId = noteIds.get(relationship.target_id) ?? null;
|
|
34679
|
+
links.push({
|
|
34680
|
+
id: aiwgShardUuid(
|
|
34681
|
+
"relationship",
|
|
34682
|
+
`${record.id}\0${position}\0${relationship.type}\0${relationship.target_id}`
|
|
34683
|
+
),
|
|
34684
|
+
from_note_id: noteIds.get(record.id),
|
|
34685
|
+
to_note_id: targetNoteId,
|
|
34686
|
+
to_url: targetNoteId ? null : `aiwg://record/${encodeURIComponent(relationship.target_id)}`,
|
|
34687
|
+
kind: relationship.type,
|
|
34688
|
+
score: relationship.confidence ?? 1,
|
|
34689
|
+
created_at: aiwgShardTimestamp(record.updated_at, createdAt2),
|
|
34690
|
+
metadata: {
|
|
34691
|
+
aiwg_fortemi_index: {
|
|
34692
|
+
source_record_id: record.id,
|
|
34693
|
+
target_record_id: relationship.target_id,
|
|
34694
|
+
relationship
|
|
34695
|
+
}
|
|
34408
34696
|
}
|
|
34409
|
-
|
|
34410
|
-
}
|
|
34697
|
+
});
|
|
34698
|
+
}
|
|
34411
34699
|
}
|
|
34412
|
-
|
|
34700
|
+
const files = /* @__PURE__ */ new Map();
|
|
34701
|
+
const components = ["notes"];
|
|
34702
|
+
const counts = {
|
|
34703
|
+
notes: notes.length
|
|
34704
|
+
};
|
|
34705
|
+
files.set("notes.jsonl", encodeJsonLines(notes));
|
|
34706
|
+
const addComponent = (component, filename, values, jsonLines = true) => {
|
|
34707
|
+
if (values.length === 0) return;
|
|
34708
|
+
components.push(component);
|
|
34709
|
+
counts[component] = values.length;
|
|
34710
|
+
files.set(filename, jsonLines ? encodeJsonLines(values) : shardEncoder.encode(JSON.stringify(values)));
|
|
34711
|
+
};
|
|
34712
|
+
addComponent("collections", "collections.json", collections, false);
|
|
34713
|
+
const tags = [...new Set(notes.flatMap((note) => note.tags))].sort().map((name) => ({ name, created_at: createdAt2 }));
|
|
34714
|
+
components.push("tags");
|
|
34715
|
+
counts.tags = tags.length;
|
|
34716
|
+
files.set("tags.json", shardEncoder.encode(JSON.stringify(tags)));
|
|
34717
|
+
addComponent("links", "links.jsonl", links);
|
|
34718
|
+
const checksums = {};
|
|
34719
|
+
for (const [filename, bytes] of files) checksums[filename] = await sha256Hex(bytes);
|
|
34720
|
+
const manifest = {
|
|
34721
|
+
version: CURRENT_SHARD_VERSION,
|
|
34722
|
+
profile: "core-v1",
|
|
34723
|
+
producer: {
|
|
34724
|
+
name: "fortemi-core-aiwg-index",
|
|
34725
|
+
version: options.matricVersion ?? "fortemi-core"
|
|
34726
|
+
},
|
|
34727
|
+
format: SHARD_FORMAT,
|
|
34728
|
+
created_at: createdAt2,
|
|
34729
|
+
components,
|
|
34730
|
+
counts: {
|
|
34731
|
+
notes: 0,
|
|
34732
|
+
collections: 0,
|
|
34733
|
+
tags: 0,
|
|
34734
|
+
templates: 0,
|
|
34735
|
+
links: 0,
|
|
34736
|
+
embedding_sets: 0,
|
|
34737
|
+
embedding_set_members: 0,
|
|
34738
|
+
embeddings: 0,
|
|
34739
|
+
embedding_configs: 0,
|
|
34740
|
+
...counts
|
|
34741
|
+
},
|
|
34742
|
+
checksums,
|
|
34743
|
+
min_reader_version: CURRENT_SHARD_VERSION
|
|
34744
|
+
};
|
|
34745
|
+
files.set("manifest.json", shardEncoder.encode(JSON.stringify(manifest, null, 2)));
|
|
34746
|
+
const contractValidation = await validateCoreV1ShardArchive(files);
|
|
34747
|
+
if (!contractValidation.valid) {
|
|
34748
|
+
throw new Error(
|
|
34749
|
+
`Generated AIWG core-v1 shard failed canonical validation: ${contractValidation.errors.join("; ")}`
|
|
34750
|
+
);
|
|
34751
|
+
}
|
|
34752
|
+
return packTarGz(files);
|
|
34413
34753
|
}
|
|
34414
|
-
function
|
|
34415
|
-
|
|
34416
|
-
|
|
34417
|
-
|
|
34418
|
-
|
|
34419
|
-
|
|
34420
|
-
const schemaRecord = { ...item };
|
|
34421
|
-
Reflect.deleteProperty(schemaRecord, "binary_sources");
|
|
34422
|
-
return schemaRecord;
|
|
34423
|
-
})
|
|
34424
|
-
} : value;
|
|
34425
|
-
const valid = validate(schemaValue);
|
|
34426
|
-
return { valid, errors: formatErrors2(validate.errors) };
|
|
34754
|
+
async function aiwgFortemiIndexToKnowledgeShardWithReport(index, options = {}) {
|
|
34755
|
+
validateAiwgIndexForShard(index);
|
|
34756
|
+
if (index.schema_version !== "aiwg.fortemi.index.export.v2") {
|
|
34757
|
+
throw new Error("Full-v1 Knowledge Shard conversion requires aiwg.fortemi.index.export.v2");
|
|
34758
|
+
}
|
|
34759
|
+
return convertAiwgIndexToFullV1(index, options);
|
|
34427
34760
|
}
|
|
34428
|
-
function
|
|
34429
|
-
const
|
|
34430
|
-
const
|
|
34431
|
-
|
|
34761
|
+
function aiwgFortemiIndexFromKnowledgeShard(bytes) {
|
|
34762
|
+
const files = unpackTarGz(bytes);
|
|
34763
|
+
const noteBytes = files.get("notes.jsonl");
|
|
34764
|
+
const notes = (noteBytes ? shardDecoder.decode(noteBytes) : "").split("\n").filter(Boolean).map((line) => JSON.parse(line));
|
|
34765
|
+
if (notes.length === 0) throw new Error("Knowledge Shard contains no AIWG index notes");
|
|
34766
|
+
let envelope;
|
|
34767
|
+
const items = [];
|
|
34768
|
+
for (const note of notes) {
|
|
34769
|
+
const metadata = note.metadata && typeof note.metadata === "object" && !Array.isArray(note.metadata) ? note.metadata.aiwg_fortemi_index : void 0;
|
|
34770
|
+
if (!metadata || typeof metadata !== "object" || Array.isArray(metadata)) {
|
|
34771
|
+
throw new Error(`Knowledge Shard note ${note.id} has no AIWG index metadata`);
|
|
34772
|
+
}
|
|
34773
|
+
const value = metadata;
|
|
34774
|
+
envelope ??= value.envelope;
|
|
34775
|
+
items.push(value.record);
|
|
34776
|
+
}
|
|
34777
|
+
const restored = {
|
|
34778
|
+
...envelope,
|
|
34779
|
+
items: items.sort((left, right) => left.id.localeCompare(right.id))
|
|
34780
|
+
};
|
|
34781
|
+
const validation = validateAiwgFortemiIndexExport(restored);
|
|
34782
|
+
if (!validation.valid) {
|
|
34783
|
+
throw new Error(`Knowledge Shard contains invalid AIWG index metadata:
|
|
34784
|
+
${validation.errors.join("\n")}`);
|
|
34785
|
+
}
|
|
34786
|
+
return restored;
|
|
34432
34787
|
}
|
|
34433
34788
|
|
|
34434
34789
|
// src/records/types.ts
|
|
@@ -36476,8 +36831,8 @@ async function importShardToRecords(store, data, options) {
|
|
|
36476
36831
|
}
|
|
36477
36832
|
|
|
36478
36833
|
// src/index.ts
|
|
36479
|
-
var VERSION = "2026.7.
|
|
36834
|
+
var VERSION = "2026.7.15";
|
|
36480
36835
|
|
|
36481
|
-
export { AIWG_SCAN_REQUIRED_FIELDS, AllowlistTrustStore, ArchiveManager, AttachmentsRepository, CORE_V1_COMPONENTS, CURRENT_MIGRATION_HEAD, CURRENT_SHARD_VERSION, CanonicalAttachmentsRepository, CanonicalNotesRepository, CapabilityManager, CaptureKnowledgeInputSchema, CollectionsRepository, CommunitiesRepository, DB_SNAPSHOT_SCHEMA_VERSION, DbSnapshotVersionError, EMBED_REQUEST_KIND, EMBED_RESPONSE_KIND, EmbeddingSetsRepository, FORTEMI_COMPATIBILITY_PATH, FORTEMI_COMPATIBILITY_STATES, FORTEMI_REQUIRED_COMPATIBILITY_CAPABILITIES, FallbackRouter, FortemiToolManifest, GetNoteInputSchema, GraphRepository, IdbRecordStore, JOB_CAPABILITIES, JOB_PRIORITIES, JobQueueWorker, LOCAL_ENDPOINTS, LinksRepository, ListNotesInputSchema, ManageArchiveInputSchema, ManageAttachmentsInputSchema, ManageCapabilitiesInputSchema, ManageCollectionsInputSchema, ManageLinksInputSchema, ManageNoteInputSchema, ManageTagsInputSchema, MemoryBlobStore, MemoryRecordStore, MigrationRunner, NotesRepository, OpenAICompatibleProvider, PGliteStorageBackend, PGliteStorageBackendFactory, PGliteWorkerClient, PGliteWorkerStorageBackend, PGliteWorkerStorageBackendFactory, ProvenanceRepository, ProviderRegistry, RECORD_COLLECTIONS, RECORD_SCHEMA_VERSION, RECORD_STORE_CAPABILITIES, SHARD_FORMAT, SIGNATURE_ENTRY, SIGNING_ENVELOPE_VERSION, SUPPORTED_PGLITE_VERSION, SearchInputSchema, SearchRepository, SkosRepository, TagsRepository, TransactionProxy, TypedEventBus, VERSION, aiRevisionHandler, aiwgFortemiIndexFromKnowledgeShard, aiwgFortemiIndexToCommunityGraph, aiwgFortemiIndexToKnowledgeShard, aiwgFortemiIndexToKnowledgeShardWithReport, allMigrations, appendPluginScript, assertAiwgFortemiChunkManifest, assertAiwgFortemiChunkPart, assertAiwgFortemiIndexExport, assertAiwgStaticEmbeddingSet, assertShardComponentRecord, buildAiwgChunkedIndex, buildAiwgStaticEmbeddingSet, buildNoteConditions, buildPluginCsp, captureKnowledge, chunkText, classifyError, classifyModel, clearPrefetchedShard, collectionFromShard, collectionToShard, computeBlobHash, computeHash, computeSri, conceptTaggingHandler, cosineSimilarity, createAiwgFetchChunkLoader, createAiwgFetchDetailLoader, createAiwgIndexController, createAiwgReviewDecisionExport, createBlobStore, createCosineSemanticProvider, createCspReportHandler, createFortemi, createLazyBlobStore, createLegacyProvider, createPGliteBackend, createPGliteInstance, createRecordBackend, createRecordStore, createRemoteBackend, createRoutes, createShardBackend, createShardCapabilityReport, createWorkerEmbedFunction, defaultStorageBackendFactory, detectCommunities, detectGpuCapabilities, detectInferenceCapabilities, discoverLocalProviders, dropAttachmentProjection, dropNoteProjection, dumpDbSnapshot, embeddingConfigToShard, embeddingFromShard, embeddingGenerationHandler, embeddingSetFromShard, embeddingSetMemberToShard, embeddingSetToShard, embeddingToShard, enqueueFullWorkflow, enqueueJob, enqueueNoteCreationJobs, estimateModelFit, estimateVramMB, estimateVramTier, exportShard, exportShardFromRecords, exportShardFromRecordsWithReport, exportShardWithReport, fetchAndValidateFortemiCompatibility, fetchPluginScript, filterAiwgRecordsByPrivacy, findAiwgStaticDuplicatePairs, formatFortemiCompatibilitySummary, fortemiCompatibilityUrl, fortemiManifest, fromPrefetched, generateId, getAiwgFortemiFacets, getAiwgFortemiIndexExportSchema, getEmbedFunction, getFortemiBridge, getFortemiSecretStore, getJobQueueStatus, getKnowledgeShardContractReceipt, getKnowledgeShardProfileRegistry, getKnowledgeShardSchema, getLlmFunction, getNote, getPrefetchedSha256, handleEmbedRequests, hasFortemiSecureSecrets, importShard, importShardToRecords, isPluginScriptAllowed, isShardPrefetched, isShardSigningSupported, linkFromShard, linkToShard, linkingHandler, listNotes, manageArchive, manageAttachments, manageCapabilities, manageCollections, manageLinks, manageNote, manageTags, matchRoute, migrateLegacyBlobStore, noteFromShard, noteSkosTagToShard, noteToShard, openShard, packTarGz, parseCspReport, prefetchShard, profileSupportError, projectAttachments, projectNotes, projectRecords, provenanceEdgeToShard, queryAiwgFortemiIndex, queryAiwgHybridIndex, queryAiwgSemanticIndex, registerLlmCapability, registerSemanticCapability, registerSemanticCapabilityWorker, registerServiceWorker, restoreDbSnapshot, searchTool, selectBackend, selectLlmModel, setEmbedFunction, setLlmFunction, sha256Hex, sidecarBlobDigests, signShard, skosConceptToShard, skosRelationToShard, skosSchemeToShard, suggestTags, tagsToShard, templateToShard, titleGenerationHandler, unpackTarGz, unregisterLlmCapability, unregisterSemanticCapability, urlLinkToShard, validateAiwgFortemiChunkManifest, validateAiwgFortemiChunkPart, validateAiwgFortemiIndexExport, validateAiwgFortemiIndexExportSchema, validateAiwgFortemiProjectedRecordSchema, validateAiwgStaticEmbeddingSet, validateChecksums, validateCoreV1ShardArchive, validateFortemiCompatibilityResponse, validateFullV1ShardArchive, validateRecordV1ShardArchive, validateShardArchive, validateShardComponentRecord, validateShardManifest, verifyDbSnapshotMeta, verifyShardSignature, verifySri };
|
|
36836
|
+
export { AIWG_SCAN_REQUIRED_FIELDS, AllowlistTrustStore, ArchiveManager, AttachmentsRepository, CORE_V1_COMPONENTS, CURRENT_MIGRATION_HEAD, CURRENT_SHARD_VERSION, CanonicalAttachmentsRepository, CanonicalNotesRepository, CapabilityManager, CaptureKnowledgeInputSchema, CollectionsRepository, CommunitiesRepository, DB_SNAPSHOT_SCHEMA_VERSION, DbSnapshotVersionError, EMBED_REQUEST_KIND, EMBED_RESPONSE_KIND, EmbeddingSetsRepository, FORTEMI_COMPATIBILITY_PATH, FORTEMI_COMPATIBILITY_STATES, FORTEMI_REQUIRED_COMPATIBILITY_CAPABILITIES, FORTEMI_SERVER_COMPATIBILITY_REVISION, FallbackRouter, FortemiToolManifest, GetNoteInputSchema, GraphRepository, IdbRecordStore, JOB_CAPABILITIES, JOB_PRIORITIES, JobQueueWorker, LOCAL_ENDPOINTS, LinksRepository, ListNotesInputSchema, ManageArchiveInputSchema, ManageAttachmentsInputSchema, ManageCapabilitiesInputSchema, ManageCollectionsInputSchema, ManageLinksInputSchema, ManageNoteInputSchema, ManageTagsInputSchema, MemoryBlobStore, MemoryRecordStore, MigrationRunner, NotesRepository, OpenAICompatibleProvider, PGliteStorageBackend, PGliteStorageBackendFactory, PGliteWorkerClient, PGliteWorkerStorageBackend, PGliteWorkerStorageBackendFactory, ProvenanceRepository, ProviderRegistry, RECORD_COLLECTIONS, RECORD_SCHEMA_VERSION, RECORD_STORE_CAPABILITIES, SHARD_FORMAT, SIGNATURE_ENTRY, SIGNING_ENVELOPE_VERSION, SUPPORTED_PGLITE_VERSION, SearchInputSchema, SearchRepository, SkosRepository, TagsRepository, TransactionProxy, TypedEventBus, VERSION, aiRevisionHandler, aiwgFortemiIndexFromKnowledgeShard, aiwgFortemiIndexToCommunityGraph, aiwgFortemiIndexToKnowledgeShard, aiwgFortemiIndexToKnowledgeShardWithReport, allMigrations, appendPluginScript, assertAiwgFortemiChunkManifest, assertAiwgFortemiChunkPart, assertAiwgFortemiIndexExport, assertAiwgStaticEmbeddingSet, assertShardComponentRecord, buildAiwgChunkedIndex, buildAiwgStaticEmbeddingSet, buildNoteConditions, buildPluginCsp, captureKnowledge, chunkText, classifyError, classifyModel, clearPrefetchedShard, collectionFromShard, collectionToShard, computeBlobHash, computeHash, computeSri, conceptTaggingHandler, cosineSimilarity, createAiwgFetchChunkLoader, createAiwgFetchDetailLoader, createAiwgIndexController, createAiwgReviewDecisionExport, createBlobStore, createCosineSemanticProvider, createCspReportHandler, createFortemi, createLazyBlobStore, createLegacyProvider, createPGliteBackend, createPGliteInstance, createRecordBackend, createRecordStore, createRemoteBackend, createRoutes, createShardBackend, createShardCapabilityReport, createWorkerEmbedFunction, defaultStorageBackendFactory, detectCommunities, detectGpuCapabilities, detectInferenceCapabilities, discoverLocalProviders, dropAttachmentProjection, dropNoteProjection, dumpDbSnapshot, embeddingConfigToShard, embeddingFromShard, embeddingGenerationHandler, embeddingSetFromShard, embeddingSetMemberToShard, embeddingSetToShard, embeddingToShard, enqueueFullWorkflow, enqueueJob, enqueueNoteCreationJobs, estimateModelFit, estimateVramMB, estimateVramTier, exportShard, exportShardFromRecords, exportShardFromRecordsWithReport, exportShardWithReport, fetchAndValidateFortemiCompatibility, fetchPluginScript, filterAiwgRecordsByPrivacy, findAiwgStaticDuplicatePairs, formatFortemiCompatibilitySummary, fortemiCompatibilityUrl, fortemiManifest, fromPrefetched, generateId, getAiwgFortemiFacets, getAiwgFortemiIndexExportSchema, getEmbedFunction, getFortemiBridge, getFortemiSecretStore, getJobQueueStatus, getKnowledgeShardContractReceipt, getKnowledgeShardProfileRegistry, getKnowledgeShardSchema, getLlmFunction, getNote, getPrefetchedSha256, handleEmbedRequests, hasFortemiSecureSecrets, importShard, importShardToRecords, isPluginScriptAllowed, isShardPrefetched, isShardSigningSupported, linkFromShard, linkToShard, linkingHandler, listNotes, manageArchive, manageAttachments, manageCapabilities, manageCollections, manageLinks, manageNote, manageTags, matchRoute, migrateLegacyBlobStore, noteFromShard, noteSkosTagToShard, noteToShard, openShard, packTarGz, parseCspReport, prefetchShard, profileSupportError, projectAttachments, projectNotes, projectRecords, provenanceEdgeToShard, queryAiwgFortemiIndex, queryAiwgHybridIndex, queryAiwgSemanticIndex, registerLlmCapability, registerSemanticCapability, registerSemanticCapabilityWorker, registerServiceWorker, restoreDbSnapshot, searchTool, selectBackend, selectLlmModel, setEmbedFunction, setLlmFunction, sha256Hex, sidecarBlobDigests, signShard, skosConceptToShard, skosRelationToShard, skosSchemeToShard, suggestTags, tagsToShard, templateToShard, titleGenerationHandler, unpackTarGz, unregisterLlmCapability, unregisterSemanticCapability, urlLinkToShard, validateAiwgFortemiChunkManifest, validateAiwgFortemiChunkPart, validateAiwgFortemiIndexExport, validateAiwgFortemiIndexExportSchema, validateAiwgFortemiProjectedRecordSchema, validateAiwgStaticEmbeddingSet, validateChecksums, validateCoreV1ShardArchive, validateFortemiCompatibilityResponse, validateFullV1ShardArchive, validateRecordV1ShardArchive, validateShardArchive, validateShardComponentRecord, validateShardManifest, verifyDbSnapshotMeta, verifyShardSignature, verifySri };
|
|
36482
36837
|
//# sourceMappingURL=index.js.map
|
|
36483
36838
|
//# sourceMappingURL=index.js.map
|