@fortemi/core 2026.5.3 → 2026.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/index.d.ts +582 -43
- package/dist/index.js +1533 -116
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -667,13 +667,134 @@ var migration0005 = {
|
|
|
667
667
|
`
|
|
668
668
|
};
|
|
669
669
|
|
|
670
|
+
// src/migrations/0006_embedding_set_metadata.ts
|
|
671
|
+
var migration0006 = {
|
|
672
|
+
version: 6,
|
|
673
|
+
name: "0006_embedding_set_metadata",
|
|
674
|
+
sql: `
|
|
675
|
+
ALTER TABLE embedding_set
|
|
676
|
+
ADD COLUMN IF NOT EXISTS name TEXT NOT NULL DEFAULT 'Full content';
|
|
677
|
+
|
|
678
|
+
ALTER TABLE embedding_set
|
|
679
|
+
ADD COLUMN IF NOT EXISTS purpose TEXT;
|
|
680
|
+
`
|
|
681
|
+
};
|
|
682
|
+
|
|
683
|
+
// src/migrations/0007_virtual_embedding_sets.ts
|
|
684
|
+
var migration0007 = {
|
|
685
|
+
version: 7,
|
|
686
|
+
name: "0007_virtual_embedding_sets",
|
|
687
|
+
sql: `
|
|
688
|
+
ALTER TABLE embedding_set
|
|
689
|
+
ADD COLUMN IF NOT EXISTS kind TEXT NOT NULL DEFAULT 'physical';
|
|
690
|
+
|
|
691
|
+
ALTER TABLE embedding_set
|
|
692
|
+
ADD COLUMN IF NOT EXISTS mode TEXT;
|
|
693
|
+
|
|
694
|
+
ALTER TABLE embedding_set
|
|
695
|
+
ADD COLUMN IF NOT EXISTS truncate_dimension INTEGER;
|
|
696
|
+
|
|
697
|
+
ALTER TABLE embedding_set
|
|
698
|
+
ADD COLUMN IF NOT EXISTS criteria_json JSONB;
|
|
699
|
+
|
|
700
|
+
ALTER TABLE embedding_set
|
|
701
|
+
ADD COLUMN IF NOT EXISTS source_json JSONB;
|
|
702
|
+
|
|
703
|
+
ALTER TABLE embedding_set
|
|
704
|
+
ADD COLUMN IF NOT EXISTS compatibility_json JSONB;
|
|
705
|
+
|
|
706
|
+
ALTER TABLE embedding_set
|
|
707
|
+
ADD COLUMN IF NOT EXISTS materialization_json JSONB;
|
|
708
|
+
|
|
709
|
+
ALTER TABLE embedding_set
|
|
710
|
+
ADD COLUMN IF NOT EXISTS freshness_json JSONB;
|
|
711
|
+
|
|
712
|
+
ALTER TABLE embedding_set
|
|
713
|
+
ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT now();
|
|
714
|
+
`
|
|
715
|
+
};
|
|
716
|
+
|
|
717
|
+
// src/migrations/0008_graph_community_artifacts.ts
|
|
718
|
+
var migration0008 = {
|
|
719
|
+
version: 8,
|
|
720
|
+
name: "0008_graph_community_artifacts",
|
|
721
|
+
sql: `
|
|
722
|
+
CREATE TABLE IF NOT EXISTS graph_source (
|
|
723
|
+
id TEXT PRIMARY KEY,
|
|
724
|
+
name TEXT NOT NULL,
|
|
725
|
+
kind TEXT NOT NULL,
|
|
726
|
+
source_table TEXT,
|
|
727
|
+
embedding_set_id TEXT,
|
|
728
|
+
virtual_set_id TEXT,
|
|
729
|
+
model TEXT,
|
|
730
|
+
dimension INTEGER,
|
|
731
|
+
truncate_dimension INTEGER,
|
|
732
|
+
metric TEXT,
|
|
733
|
+
algorithm TEXT,
|
|
734
|
+
parameters_json JSONB,
|
|
735
|
+
input_hash TEXT NOT NULL,
|
|
736
|
+
freshness_json JSONB NOT NULL,
|
|
737
|
+
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
|
738
|
+
);
|
|
739
|
+
|
|
740
|
+
CREATE TABLE IF NOT EXISTS graph_edge_artifact (
|
|
741
|
+
graph_source_id TEXT NOT NULL REFERENCES graph_source(id) ON DELETE CASCADE,
|
|
742
|
+
from_note_id TEXT NOT NULL REFERENCES note(id),
|
|
743
|
+
to_note_id TEXT NOT NULL REFERENCES note(id),
|
|
744
|
+
weight DOUBLE PRECISION NOT NULL,
|
|
745
|
+
kind TEXT NOT NULL,
|
|
746
|
+
rank INTEGER,
|
|
747
|
+
metadata_json JSONB,
|
|
748
|
+
PRIMARY KEY (graph_source_id, from_note_id, to_note_id, kind)
|
|
749
|
+
);
|
|
750
|
+
|
|
751
|
+
CREATE TABLE IF NOT EXISTS community_set (
|
|
752
|
+
id TEXT PRIMARY KEY,
|
|
753
|
+
graph_source_id TEXT NOT NULL REFERENCES graph_source(id) ON DELETE CASCADE,
|
|
754
|
+
name TEXT NOT NULL,
|
|
755
|
+
source_type TEXT NOT NULL,
|
|
756
|
+
algorithm TEXT,
|
|
757
|
+
parameters_json JSONB,
|
|
758
|
+
input_hash TEXT NOT NULL,
|
|
759
|
+
freshness_json JSONB NOT NULL,
|
|
760
|
+
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
|
761
|
+
);
|
|
762
|
+
|
|
763
|
+
CREATE TABLE IF NOT EXISTS community (
|
|
764
|
+
id TEXT NOT NULL,
|
|
765
|
+
community_set_id TEXT NOT NULL REFERENCES community_set(id) ON DELETE CASCADE,
|
|
766
|
+
label TEXT,
|
|
767
|
+
rank INTEGER,
|
|
768
|
+
size INTEGER,
|
|
769
|
+
confidence DOUBLE PRECISION,
|
|
770
|
+
representative_note_ids TEXT[],
|
|
771
|
+
metadata_json JSONB,
|
|
772
|
+
PRIMARY KEY (community_set_id, id)
|
|
773
|
+
);
|
|
774
|
+
|
|
775
|
+
CREATE TABLE IF NOT EXISTS community_assignment (
|
|
776
|
+
community_set_id TEXT NOT NULL REFERENCES community_set(id) ON DELETE CASCADE,
|
|
777
|
+
community_id TEXT NOT NULL,
|
|
778
|
+
note_id TEXT NOT NULL REFERENCES note(id),
|
|
779
|
+
confidence DOUBLE PRECISION,
|
|
780
|
+
source_type TEXT NOT NULL,
|
|
781
|
+
metadata_json JSONB,
|
|
782
|
+
PRIMARY KEY (community_set_id, note_id),
|
|
783
|
+
FOREIGN KEY (community_set_id, community_id) REFERENCES community(community_set_id, id) ON DELETE CASCADE
|
|
784
|
+
);
|
|
785
|
+
`
|
|
786
|
+
};
|
|
787
|
+
|
|
670
788
|
// src/migrations/index.ts
|
|
671
789
|
var allMigrations = [
|
|
672
790
|
migration0001,
|
|
673
791
|
migration0002,
|
|
674
792
|
migration0003,
|
|
675
793
|
migration0004,
|
|
676
|
-
migration0005
|
|
794
|
+
migration0005,
|
|
795
|
+
migration0006,
|
|
796
|
+
migration0007,
|
|
797
|
+
migration0008
|
|
677
798
|
];
|
|
678
799
|
|
|
679
800
|
// src/archive-manager.ts
|
|
@@ -1547,26 +1668,410 @@ function buildNoteConditions(options, startIdx, includeDeleted = false) {
|
|
|
1547
1668
|
return { conditions, params, nextIdx: idx };
|
|
1548
1669
|
}
|
|
1549
1670
|
|
|
1671
|
+
// src/repositories/embedding-sets-repository.ts
|
|
1672
|
+
var DEFAULT_COMPATIBILITY = {
|
|
1673
|
+
model: "require-same",
|
|
1674
|
+
dimension: "require-same",
|
|
1675
|
+
duplicateVectors: "prefer-set-order",
|
|
1676
|
+
missingVectors: "omit"
|
|
1677
|
+
};
|
|
1678
|
+
function jsonParam(value) {
|
|
1679
|
+
return value == null ? null : JSON.stringify(value);
|
|
1680
|
+
}
|
|
1681
|
+
function asObject(value) {
|
|
1682
|
+
if (value == null) return null;
|
|
1683
|
+
if (typeof value === "string") return JSON.parse(value);
|
|
1684
|
+
return value;
|
|
1685
|
+
}
|
|
1686
|
+
function dateString(value) {
|
|
1687
|
+
if (!value) return void 0;
|
|
1688
|
+
return value instanceof Date ? value.toISOString() : value;
|
|
1689
|
+
}
|
|
1690
|
+
function dateMillis(value) {
|
|
1691
|
+
return value instanceof Date ? value.getTime() : new Date(value).getTime();
|
|
1692
|
+
}
|
|
1693
|
+
var EmbeddingSetsRepository = class {
|
|
1694
|
+
constructor(db) {
|
|
1695
|
+
this.db = db;
|
|
1696
|
+
}
|
|
1697
|
+
async create(input) {
|
|
1698
|
+
const id = input.id ?? generateId();
|
|
1699
|
+
await this.db.query(
|
|
1700
|
+
`INSERT INTO embedding_set (
|
|
1701
|
+
id, name, purpose, model_name, dimensions, kind, mode,
|
|
1702
|
+
truncate_dimension, criteria_json
|
|
1703
|
+
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::jsonb)`,
|
|
1704
|
+
[
|
|
1705
|
+
id,
|
|
1706
|
+
input.name,
|
|
1707
|
+
input.purpose ?? null,
|
|
1708
|
+
input.model_name ?? "all-MiniLM-L6-v2",
|
|
1709
|
+
input.dimensions ?? 384,
|
|
1710
|
+
input.kind ?? "physical",
|
|
1711
|
+
input.mode ?? null,
|
|
1712
|
+
input.truncate_dimension ?? null,
|
|
1713
|
+
jsonParam(input.criteria ?? null)
|
|
1714
|
+
]
|
|
1715
|
+
);
|
|
1716
|
+
return this.get(id);
|
|
1717
|
+
}
|
|
1718
|
+
async createVirtualDefinition(input) {
|
|
1719
|
+
const id = input.id ?? generateId();
|
|
1720
|
+
await this.db.query(
|
|
1721
|
+
`INSERT INTO embedding_set (
|
|
1722
|
+
id, name, purpose, model_name, dimensions, kind, mode,
|
|
1723
|
+
source_json, compatibility_json, materialization_json, freshness_json, created_at, updated_at
|
|
1724
|
+
) VALUES ($1, $2, $3, $4, $5, 'virtual', 'auto', $6::jsonb, $7::jsonb, $8::jsonb, $9::jsonb,
|
|
1725
|
+
COALESCE($10::timestamptz, now()), COALESCE($11::timestamptz, now()))`,
|
|
1726
|
+
[
|
|
1727
|
+
id,
|
|
1728
|
+
input.name,
|
|
1729
|
+
input.purpose ?? null,
|
|
1730
|
+
this.inferDefinitionModel(input) ?? "virtual",
|
|
1731
|
+
this.inferDefinitionDimension(input) ?? 0,
|
|
1732
|
+
jsonParam(input.source),
|
|
1733
|
+
jsonParam(input.compatibility),
|
|
1734
|
+
jsonParam(input.materialization ?? null),
|
|
1735
|
+
jsonParam({ status: input.materialization?.freshness ?? "unknown" }),
|
|
1736
|
+
input.createdAt ?? null,
|
|
1737
|
+
input.updatedAt ?? null
|
|
1738
|
+
]
|
|
1739
|
+
);
|
|
1740
|
+
return this.get(id);
|
|
1741
|
+
}
|
|
1742
|
+
async ensureDefault() {
|
|
1743
|
+
const existing = await this.db.query(
|
|
1744
|
+
`SELECT * FROM embedding_set WHERE name = $1 AND model_name = $2 AND kind = 'physical' ORDER BY created_at LIMIT 1`,
|
|
1745
|
+
["Full content", "all-MiniLM-L6-v2"]
|
|
1746
|
+
);
|
|
1747
|
+
if (existing.rows.length > 0) return existing.rows[0];
|
|
1748
|
+
return this.create({
|
|
1749
|
+
name: "Full content",
|
|
1750
|
+
purpose: "Semantic search over full revised note content",
|
|
1751
|
+
model_name: "all-MiniLM-L6-v2",
|
|
1752
|
+
dimensions: 384,
|
|
1753
|
+
kind: "physical"
|
|
1754
|
+
});
|
|
1755
|
+
}
|
|
1756
|
+
async get(id) {
|
|
1757
|
+
const result = await this.db.query(
|
|
1758
|
+
`SELECT * FROM embedding_set WHERE id = $1`,
|
|
1759
|
+
[id]
|
|
1760
|
+
);
|
|
1761
|
+
if (result.rows.length === 0) throw new Error(`Embedding set not found: ${id}`);
|
|
1762
|
+
return result.rows[0];
|
|
1763
|
+
}
|
|
1764
|
+
async list() {
|
|
1765
|
+
const result = await this.db.query(
|
|
1766
|
+
`SELECT * FROM embedding_set ORDER BY created_at, name`
|
|
1767
|
+
);
|
|
1768
|
+
return result.rows;
|
|
1769
|
+
}
|
|
1770
|
+
async listDescriptors() {
|
|
1771
|
+
const rows = await this.list();
|
|
1772
|
+
return rows.map((row) => this.toDescriptor(row));
|
|
1773
|
+
}
|
|
1774
|
+
toDescriptor(row) {
|
|
1775
|
+
return {
|
|
1776
|
+
id: row.id,
|
|
1777
|
+
name: row.name,
|
|
1778
|
+
purpose: row.purpose,
|
|
1779
|
+
kind: row.kind,
|
|
1780
|
+
mode: row.mode ?? void 0,
|
|
1781
|
+
model: row.model_name,
|
|
1782
|
+
dimension: row.dimensions,
|
|
1783
|
+
truncateDimension: row.truncate_dimension,
|
|
1784
|
+
criteria: asObject(row.criteria_json),
|
|
1785
|
+
createdAt: dateString(row.created_at),
|
|
1786
|
+
updatedAt: dateString(row.updated_at),
|
|
1787
|
+
freshness: asObject(row.freshness_json) ?? { status: "fresh" }
|
|
1788
|
+
};
|
|
1789
|
+
}
|
|
1790
|
+
async putEmbedding(input) {
|
|
1791
|
+
const set = await this.get(input.embedding_set_id);
|
|
1792
|
+
if (set.kind === "virtual") {
|
|
1793
|
+
throw new Error(`Cannot store vectors directly in virtual embedding set: ${set.id}`);
|
|
1794
|
+
}
|
|
1795
|
+
if (input.vector.length !== set.dimensions) {
|
|
1796
|
+
throw new Error(
|
|
1797
|
+
`Embedding vector has ${input.vector.length} dimensions; set ${set.id} expects ${set.dimensions}`
|
|
1798
|
+
);
|
|
1799
|
+
}
|
|
1800
|
+
await this.db.query(
|
|
1801
|
+
`DELETE FROM embedding_set_member WHERE note_id = $1 AND embedding_set_id = $2`,
|
|
1802
|
+
[input.note_id, input.embedding_set_id]
|
|
1803
|
+
);
|
|
1804
|
+
await this.db.query(
|
|
1805
|
+
`DELETE FROM embedding WHERE note_id = $1 AND embedding_set_id = $2`,
|
|
1806
|
+
[input.note_id, input.embedding_set_id]
|
|
1807
|
+
);
|
|
1808
|
+
const embeddingId = input.id ?? generateId();
|
|
1809
|
+
const vector2 = `[${input.vector.join(",")}]`;
|
|
1810
|
+
await this.db.query(
|
|
1811
|
+
`INSERT INTO embedding (id, note_id, embedding_set_id, vector)
|
|
1812
|
+
VALUES ($1, $2, $3, $4::vector)`,
|
|
1813
|
+
[embeddingId, input.note_id, input.embedding_set_id, vector2]
|
|
1814
|
+
);
|
|
1815
|
+
await this.db.query(
|
|
1816
|
+
`INSERT INTO embedding_set_member (embedding_set_id, note_id, embedding_id)
|
|
1817
|
+
VALUES ($1, $2, $3)`,
|
|
1818
|
+
[input.embedding_set_id, input.note_id, embeddingId]
|
|
1819
|
+
);
|
|
1820
|
+
return { id: embeddingId };
|
|
1821
|
+
}
|
|
1822
|
+
async resolveSelector(selector) {
|
|
1823
|
+
if (selector.kind === "default") {
|
|
1824
|
+
const set = await this.ensureDefault();
|
|
1825
|
+
return this.resolvePhysicalSet({ kind: "embedding-set", embeddingSetId: set.id }, set.id);
|
|
1826
|
+
}
|
|
1827
|
+
if (selector.kind === "embedding-set") {
|
|
1828
|
+
if (!selector.embeddingSetId) throw new Error("embedding-set selector requires embeddingSetId");
|
|
1829
|
+
const set = await this.get(selector.embeddingSetId);
|
|
1830
|
+
if (set.kind === "virtual") {
|
|
1831
|
+
const definition = this.definitionFromRow(set);
|
|
1832
|
+
return this.resolveDefinition({ kind: "embedding-set", embeddingSetId: set.id }, definition);
|
|
1833
|
+
}
|
|
1834
|
+
return this.resolvePhysicalSet(selector, set.id);
|
|
1835
|
+
}
|
|
1836
|
+
if (!selector.definition) throw new Error("virtual-definition selector requires definition");
|
|
1837
|
+
return this.resolveDefinition(selector, selector.definition);
|
|
1838
|
+
}
|
|
1839
|
+
async resolveDefinition(selector, definition) {
|
|
1840
|
+
let rows;
|
|
1841
|
+
const errors = [];
|
|
1842
|
+
switch (definition.source.type) {
|
|
1843
|
+
case "criteria":
|
|
1844
|
+
rows = await this.resolveCriteriaSource(definition.source);
|
|
1845
|
+
break;
|
|
1846
|
+
case "set-operation":
|
|
1847
|
+
rows = await this.resolveSetOperationSource(definition.source, definition.compatibility, errors);
|
|
1848
|
+
break;
|
|
1849
|
+
case "fallback":
|
|
1850
|
+
rows = await this.resolveFallbackSource(definition.source.preferredSetIds, definition.compatibility, errors);
|
|
1851
|
+
break;
|
|
1852
|
+
case "latest-compatible":
|
|
1853
|
+
rows = await this.resolveLatestCompatibleSource(definition.source, definition.compatibility, errors);
|
|
1854
|
+
break;
|
|
1855
|
+
case "snapshot":
|
|
1856
|
+
rows = await this.resolvePhysicalRows(definition.source.snapshotId);
|
|
1857
|
+
break;
|
|
1858
|
+
default:
|
|
1859
|
+
rows = [];
|
|
1860
|
+
}
|
|
1861
|
+
return this.finalizeResolution(selector, rows, errors, definition.compatibility, definition.materialization?.freshness ?? "unknown");
|
|
1862
|
+
}
|
|
1863
|
+
async resolvePhysicalSet(selector, setId) {
|
|
1864
|
+
return this.finalizeResolution(selector, await this.resolvePhysicalRows(setId), [], DEFAULT_COMPATIBILITY, "fresh");
|
|
1865
|
+
}
|
|
1866
|
+
async resolvePhysicalRows(setId) {
|
|
1867
|
+
const result = await this.db.query(
|
|
1868
|
+
`SELECT note_id, embedding_set_id, id as embedding_id, vector::text as vector, created_at
|
|
1869
|
+
FROM embedding
|
|
1870
|
+
WHERE embedding_set_id = $1
|
|
1871
|
+
ORDER BY note_id, created_at DESC`,
|
|
1872
|
+
[setId]
|
|
1873
|
+
);
|
|
1874
|
+
return result.rows;
|
|
1875
|
+
}
|
|
1876
|
+
async resolveCriteriaSource(source) {
|
|
1877
|
+
const criteria = source.criteria;
|
|
1878
|
+
if (criteria.conceptIds && criteria.conceptIds.length > 0) {
|
|
1879
|
+
throw new Error("Unsupported virtual embedding-set criteria field: conceptIds");
|
|
1880
|
+
}
|
|
1881
|
+
const conditions = ["e.embedding_set_id = $1"];
|
|
1882
|
+
const params = [source.baseSetId];
|
|
1883
|
+
let idx = 2;
|
|
1884
|
+
if (criteria.noteIds?.length) {
|
|
1885
|
+
conditions.push(`n.id = ANY($${idx++})`);
|
|
1886
|
+
params.push(criteria.noteIds);
|
|
1887
|
+
}
|
|
1888
|
+
if (criteria.tags?.length) {
|
|
1889
|
+
conditions.push(`EXISTS (SELECT 1 FROM note_tag nt WHERE nt.note_id = n.id AND nt.tag = ANY($${idx++}))`);
|
|
1890
|
+
params.push(criteria.tags);
|
|
1891
|
+
}
|
|
1892
|
+
if (criteria.collectionIds?.length) {
|
|
1893
|
+
conditions.push(`EXISTS (SELECT 1 FROM collection_note cn WHERE cn.note_id = n.id AND cn.collection_id = ANY($${idx++}))`);
|
|
1894
|
+
params.push(criteria.collectionIds);
|
|
1895
|
+
}
|
|
1896
|
+
if (criteria.updatedAfter) {
|
|
1897
|
+
conditions.push(`n.updated_at >= $${idx++}`);
|
|
1898
|
+
params.push(criteria.updatedAfter);
|
|
1899
|
+
}
|
|
1900
|
+
if (criteria.updatedBefore) {
|
|
1901
|
+
conditions.push(`n.updated_at <= $${idx++}`);
|
|
1902
|
+
params.push(criteria.updatedBefore);
|
|
1903
|
+
}
|
|
1904
|
+
if (criteria.query?.trim()) {
|
|
1905
|
+
conditions.push(`(n.tsv @@ plainto_tsquery('english', $${idx}) OR to_tsvector('english', coalesce(c.content, '')) @@ plainto_tsquery('english', $${idx}))`);
|
|
1906
|
+
params.push(criteria.query);
|
|
1907
|
+
}
|
|
1908
|
+
const result = await this.db.query(
|
|
1909
|
+
`SELECT e.note_id, e.embedding_set_id, e.id as embedding_id, e.vector::text as vector, e.created_at
|
|
1910
|
+
FROM embedding e
|
|
1911
|
+
JOIN note n ON n.id = e.note_id
|
|
1912
|
+
LEFT JOIN note_revised_current c ON c.note_id = n.id
|
|
1913
|
+
WHERE ${conditions.join(" AND ")}
|
|
1914
|
+
ORDER BY e.note_id, e.created_at DESC`,
|
|
1915
|
+
params
|
|
1916
|
+
);
|
|
1917
|
+
return result.rows;
|
|
1918
|
+
}
|
|
1919
|
+
async resolveSetOperationSource(source, compatibility, errors) {
|
|
1920
|
+
const bySet = /* @__PURE__ */ new Map();
|
|
1921
|
+
for (const setId of source.setIds) bySet.set(setId, await this.resolvePhysicalRows(setId));
|
|
1922
|
+
await this.validateCompatibility(source.setIds, compatibility, errors);
|
|
1923
|
+
const noteSets = source.setIds.map((setId) => new Set((bySet.get(setId) ?? []).map((row) => row.note_id)));
|
|
1924
|
+
const firstRows = bySet.get(source.setIds[0]) ?? [];
|
|
1925
|
+
if (source.operation === "difference") {
|
|
1926
|
+
const excluded = new Set(noteSets.slice(1).flatMap((set) => Array.from(set)));
|
|
1927
|
+
return firstRows.filter((row) => !excluded.has(row.note_id));
|
|
1928
|
+
}
|
|
1929
|
+
if (source.operation === "intersection") {
|
|
1930
|
+
return firstRows.filter((row) => noteSets.every((set) => set.has(row.note_id)));
|
|
1931
|
+
}
|
|
1932
|
+
return this.resolveDuplicateRows(source.setIds.flatMap((setId) => bySet.get(setId) ?? []), compatibility, errors);
|
|
1933
|
+
}
|
|
1934
|
+
async resolveFallbackSource(setIds, compatibility, errors) {
|
|
1935
|
+
await this.validateCompatibility(setIds, compatibility, errors);
|
|
1936
|
+
const selected = /* @__PURE__ */ new Map();
|
|
1937
|
+
for (const setId of setIds) {
|
|
1938
|
+
for (const row of await this.resolvePhysicalRows(setId)) {
|
|
1939
|
+
if (!selected.has(row.note_id)) selected.set(row.note_id, row);
|
|
1940
|
+
}
|
|
1941
|
+
}
|
|
1942
|
+
return Array.from(selected.values()).sort((a, b) => a.note_id.localeCompare(b.note_id));
|
|
1943
|
+
}
|
|
1944
|
+
async resolveLatestCompatibleSource(source, compatibility, errors) {
|
|
1945
|
+
const sets = [];
|
|
1946
|
+
for (const setId of source.candidateSetIds) {
|
|
1947
|
+
const set = await this.get(setId);
|
|
1948
|
+
if (source.model && set.model_name !== source.model) continue;
|
|
1949
|
+
if (source.dimension && set.dimensions !== source.dimension) continue;
|
|
1950
|
+
sets.push(set);
|
|
1951
|
+
}
|
|
1952
|
+
sets.sort((a, b) => dateMillis(b.updated_at) - dateMillis(a.updated_at) || dateMillis(b.created_at) - dateMillis(a.created_at));
|
|
1953
|
+
return this.resolveFallbackSource(sets.map((set) => set.id), compatibility, errors);
|
|
1954
|
+
}
|
|
1955
|
+
async validateCompatibility(setIds, compatibility, errors) {
|
|
1956
|
+
const sets = [];
|
|
1957
|
+
for (const setId of setIds) sets.push(await this.get(setId));
|
|
1958
|
+
if (compatibility.model === "require-same" && new Set(sets.map((set) => set.model_name)).size > 1) {
|
|
1959
|
+
errors.push({ code: "mixed-models", setIds });
|
|
1960
|
+
}
|
|
1961
|
+
if (compatibility.dimension === "require-same" && new Set(sets.map((set) => set.dimensions)).size > 1) {
|
|
1962
|
+
errors.push({ code: "mixed-dimensions", setIds });
|
|
1963
|
+
}
|
|
1964
|
+
}
|
|
1965
|
+
resolveDuplicateRows(rows, compatibility, errors) {
|
|
1966
|
+
const byNote = /* @__PURE__ */ new Map();
|
|
1967
|
+
for (const row of rows) {
|
|
1968
|
+
const existing = byNote.get(row.note_id) ?? [];
|
|
1969
|
+
existing.push(row);
|
|
1970
|
+
byNote.set(row.note_id, existing);
|
|
1971
|
+
}
|
|
1972
|
+
const resolved = [];
|
|
1973
|
+
for (const [noteId, noteRows] of byNote) {
|
|
1974
|
+
if (noteRows.length > 1 && compatibility.duplicateVectors === "error") {
|
|
1975
|
+
errors.push({ code: "duplicate-vector", noteId, setIds: noteRows.map((row) => row.embedding_set_id) });
|
|
1976
|
+
continue;
|
|
1977
|
+
}
|
|
1978
|
+
const ordered = [...noteRows];
|
|
1979
|
+
if (compatibility.duplicateVectors === "prefer-latest") {
|
|
1980
|
+
ordered.sort((a, b) => dateMillis(b.created_at) - dateMillis(a.created_at));
|
|
1981
|
+
}
|
|
1982
|
+
resolved.push(ordered[0]);
|
|
1983
|
+
}
|
|
1984
|
+
return resolved.sort((a, b) => a.note_id.localeCompare(b.note_id));
|
|
1985
|
+
}
|
|
1986
|
+
finalizeResolution(selector, rows, errors, compatibility, freshness) {
|
|
1987
|
+
const deduped = this.resolveDuplicateRows(rows, compatibility, errors);
|
|
1988
|
+
return {
|
|
1989
|
+
selector,
|
|
1990
|
+
rows: deduped,
|
|
1991
|
+
noteIds: deduped.map((row) => row.note_id),
|
|
1992
|
+
embeddingIds: deduped.map((row) => row.embedding_id),
|
|
1993
|
+
errors,
|
|
1994
|
+
freshness: { status: freshness }
|
|
1995
|
+
};
|
|
1996
|
+
}
|
|
1997
|
+
definitionFromRow(row) {
|
|
1998
|
+
const source = asObject(row.source_json);
|
|
1999
|
+
if (!source) throw new Error(`Virtual embedding set has no source definition: ${row.id}`);
|
|
2000
|
+
return {
|
|
2001
|
+
id: row.id,
|
|
2002
|
+
name: row.name,
|
|
2003
|
+
purpose: row.purpose,
|
|
2004
|
+
source,
|
|
2005
|
+
compatibility: asObject(row.compatibility_json) ?? DEFAULT_COMPATIBILITY,
|
|
2006
|
+
materialization: asObject(row.materialization_json) ?? void 0,
|
|
2007
|
+
createdAt: dateString(row.created_at),
|
|
2008
|
+
updatedAt: dateString(row.updated_at)
|
|
2009
|
+
};
|
|
2010
|
+
}
|
|
2011
|
+
inferDefinitionModel(input) {
|
|
2012
|
+
if (input.source.type === "latest-compatible") return input.source.model ?? null;
|
|
2013
|
+
return null;
|
|
2014
|
+
}
|
|
2015
|
+
inferDefinitionDimension(input) {
|
|
2016
|
+
if (input.source.type === "latest-compatible") return input.source.dimension ?? null;
|
|
2017
|
+
return null;
|
|
2018
|
+
}
|
|
2019
|
+
};
|
|
2020
|
+
|
|
1550
2021
|
// src/repositories/search-repository.ts
|
|
1551
2022
|
var SearchRepository = class {
|
|
1552
2023
|
constructor(db, semanticAvailable = false) {
|
|
1553
2024
|
this.db = db;
|
|
1554
2025
|
this.semanticAvailable = semanticAvailable;
|
|
1555
2026
|
}
|
|
1556
|
-
/** Select tsquery function based on whether query contains quoted phrases */
|
|
1557
2027
|
tsqueryFn(query) {
|
|
1558
2028
|
return query.includes('"') ? "phraseto_tsquery" : "plainto_tsquery";
|
|
1559
2029
|
}
|
|
1560
|
-
|
|
1561
|
-
async fetchEmbeddingSet(noteIds) {
|
|
2030
|
+
async fetchEmbeddingSet(noteIds, embeddingSetId) {
|
|
1562
2031
|
if (noteIds.length === 0) return /* @__PURE__ */ new Set();
|
|
2032
|
+
const params = [noteIds];
|
|
2033
|
+
const setFilter = embeddingSetId ? " AND embedding_set_id = $2" : "";
|
|
2034
|
+
if (embeddingSetId) params.push(embeddingSetId);
|
|
1563
2035
|
const result = await this.db.query(
|
|
1564
|
-
|
|
1565
|
-
|
|
2036
|
+
"SELECT note_id FROM embedding WHERE note_id = ANY($1)" + setFilter,
|
|
2037
|
+
params
|
|
1566
2038
|
);
|
|
1567
2039
|
return new Set(result.rows.map((r) => r.note_id));
|
|
1568
2040
|
}
|
|
1569
|
-
|
|
2041
|
+
selectorFromOptions(options) {
|
|
2042
|
+
if (options.embeddingSetSelector) return options.embeddingSetSelector;
|
|
2043
|
+
if (options.embeddingSetId) return { kind: "embedding-set", embeddingSetId: options.embeddingSetId };
|
|
2044
|
+
return null;
|
|
2045
|
+
}
|
|
2046
|
+
async resolveEmbeddingSet(options) {
|
|
2047
|
+
const selector = this.selectorFromOptions(options);
|
|
2048
|
+
if (!selector) return null;
|
|
2049
|
+
return new EmbeddingSetsRepository(this.db).resolveSelector(selector);
|
|
2050
|
+
}
|
|
2051
|
+
scopeToResolvedEmbeddingSet(conditions, params, paramIdx, resolved) {
|
|
2052
|
+
if (!resolved) return paramIdx;
|
|
2053
|
+
if (resolved.noteIds.length === 0) {
|
|
2054
|
+
conditions.push("FALSE");
|
|
2055
|
+
return paramIdx;
|
|
2056
|
+
}
|
|
2057
|
+
conditions.push("n.id = ANY($" + paramIdx + ")");
|
|
2058
|
+
params.push(resolved.noteIds);
|
|
2059
|
+
return paramIdx + 1;
|
|
2060
|
+
}
|
|
2061
|
+
scopeToResolvedEmbeddingRows(conditions, params, paramIdx, resolved) {
|
|
2062
|
+
if (!resolved) return paramIdx;
|
|
2063
|
+
if (resolved.embeddingIds.length === 0) {
|
|
2064
|
+
conditions.push("FALSE");
|
|
2065
|
+
return paramIdx;
|
|
2066
|
+
}
|
|
2067
|
+
conditions.push("e.id = ANY($" + paramIdx + ")");
|
|
2068
|
+
params.push(resolved.embeddingIds);
|
|
2069
|
+
return paramIdx + 1;
|
|
2070
|
+
}
|
|
2071
|
+
async fetchEmbeddingStatus(noteIds, resolved, embeddingSetId) {
|
|
2072
|
+
if (resolved) return new Set(noteIds.filter((id) => resolved.noteIds.includes(id)));
|
|
2073
|
+
return this.fetchEmbeddingSet(noteIds, embeddingSetId);
|
|
2074
|
+
}
|
|
1570
2075
|
attachEmbeddingStatus(results, embeddingSet) {
|
|
1571
2076
|
return results.map((r) => ({ ...r, has_embedding: embeddingSet.has(r.id) }));
|
|
1572
2077
|
}
|
|
@@ -1574,9 +2079,7 @@ var SearchRepository = class {
|
|
|
1574
2079
|
const { limit = 20, offset = 0 } = options;
|
|
1575
2080
|
const mode = options.mode ?? "auto";
|
|
1576
2081
|
if (mode === "text") {
|
|
1577
|
-
if (!query.trim())
|
|
1578
|
-
return this.recentNotes(options);
|
|
1579
|
-
}
|
|
2082
|
+
if (!query.trim()) return this.recentNotes(options);
|
|
1580
2083
|
} else if (mode === "semantic") {
|
|
1581
2084
|
if (!queryEmbedding || queryEmbedding.length === 0) {
|
|
1582
2085
|
throw new Error("mode=semantic requires a query embedding");
|
|
@@ -1589,26 +2092,21 @@ var SearchRepository = class {
|
|
|
1589
2092
|
return this.hybridSearch(query, queryEmbedding, options);
|
|
1590
2093
|
} else {
|
|
1591
2094
|
if (queryEmbedding && queryEmbedding.length > 0) {
|
|
1592
|
-
if (query.trim())
|
|
1593
|
-
return this.hybridSearch(query, queryEmbedding, options);
|
|
1594
|
-
}
|
|
2095
|
+
if (query.trim()) return this.hybridSearch(query, queryEmbedding, options);
|
|
1595
2096
|
return this.semanticSearch(queryEmbedding, options);
|
|
1596
2097
|
}
|
|
1597
|
-
if (!query.trim())
|
|
1598
|
-
return this.recentNotes(options);
|
|
1599
|
-
}
|
|
1600
|
-
}
|
|
1601
|
-
if (!query.trim()) {
|
|
1602
|
-
return this.recentNotes(options);
|
|
2098
|
+
if (!query.trim()) return this.recentNotes(options);
|
|
1603
2099
|
}
|
|
2100
|
+
if (!query.trim()) return this.recentNotes(options);
|
|
2101
|
+
const resolvedEmbeddingSet = await this.resolveEmbeddingSet(options);
|
|
1604
2102
|
const tsqFn = this.tsqueryFn(query);
|
|
1605
2103
|
const { conditions, params, nextIdx } = buildNoteConditions(options, 2);
|
|
1606
2104
|
conditions.unshift(
|
|
1607
2105
|
`(n.tsv @@ ${tsqFn}('english', $1) OR
|
|
1608
2106
|
to_tsvector('english', coalesce(c.content, '')) @@ ${tsqFn}('english', $1))`
|
|
1609
2107
|
);
|
|
2108
|
+
let paramIdx = this.scopeToResolvedEmbeddingSet(conditions, params, nextIdx, resolvedEmbeddingSet);
|
|
1610
2109
|
const allParams = [query, ...params];
|
|
1611
|
-
let paramIdx = nextIdx;
|
|
1612
2110
|
const where = conditions.join(" AND ");
|
|
1613
2111
|
const countResult = await this.db.query(
|
|
1614
2112
|
`SELECT COUNT(*) as count
|
|
@@ -1641,7 +2139,7 @@ var SearchRepository = class {
|
|
|
1641
2139
|
const resultIds = result.rows.map((r) => r.id);
|
|
1642
2140
|
const [tagMap, embeddingSet] = await Promise.all([
|
|
1643
2141
|
this.fetchTagMap(resultIds),
|
|
1644
|
-
this.
|
|
2142
|
+
this.fetchEmbeddingStatus(resultIds, resolvedEmbeddingSet, options.embeddingSetId)
|
|
1645
2143
|
]);
|
|
1646
2144
|
let facets;
|
|
1647
2145
|
if (options.include_facets) {
|
|
@@ -1673,15 +2171,12 @@ var SearchRepository = class {
|
|
|
1673
2171
|
facets
|
|
1674
2172
|
};
|
|
1675
2173
|
}
|
|
1676
|
-
/**
|
|
1677
|
-
* Semantic search using pgvector cosine distance.
|
|
1678
|
-
* Returns notes ranked by vector similarity to the query embedding.
|
|
1679
|
-
*/
|
|
1680
2174
|
async semanticSearch(queryEmbedding, options = {}) {
|
|
1681
2175
|
const { limit = 20, offset = 0 } = options;
|
|
1682
2176
|
const vectorStr = `[${queryEmbedding.join(",")}]`;
|
|
2177
|
+
const resolvedEmbeddingSet = await this.resolveEmbeddingSet(options);
|
|
1683
2178
|
const { conditions, params, nextIdx } = buildNoteConditions(options, 1);
|
|
1684
|
-
let paramIdx = nextIdx;
|
|
2179
|
+
let paramIdx = this.scopeToResolvedEmbeddingRows(conditions, params, nextIdx, resolvedEmbeddingSet);
|
|
1685
2180
|
const where = conditions.join(" AND ");
|
|
1686
2181
|
const countResult = await this.db.query(
|
|
1687
2182
|
`SELECT COUNT(*) as count
|
|
@@ -1707,26 +2202,20 @@ var SearchRepository = class {
|
|
|
1707
2202
|
[...params, vectorStr, limit, offset]
|
|
1708
2203
|
);
|
|
1709
2204
|
const tagMap = await this.fetchTagMap(result.rows.map((r) => r.id));
|
|
1710
|
-
|
|
1711
|
-
|
|
1712
|
-
|
|
1713
|
-
|
|
1714
|
-
params
|
|
1715
|
-
);
|
|
1716
|
-
facets = await this.fetchFacets(idsResult.rows.map((r) => r.id));
|
|
1717
|
-
}
|
|
2205
|
+
const facets = options.include_facets ? await this.fetchFacets((await this.db.query(
|
|
2206
|
+
`SELECT n.id FROM embedding e JOIN note n ON n.id = e.note_id WHERE ${where}`,
|
|
2207
|
+
params
|
|
2208
|
+
)).rows.map((r) => r.id)) : void 0;
|
|
1718
2209
|
return {
|
|
1719
2210
|
results: result.rows.map((r) => ({
|
|
1720
2211
|
id: r.id,
|
|
1721
2212
|
title: r.title,
|
|
1722
2213
|
snippet: r.snippet ?? "",
|
|
1723
2214
|
rank: 1 - r.distance,
|
|
1724
|
-
// Convert distance to similarity score
|
|
1725
2215
|
created_at: r.created_at,
|
|
1726
2216
|
updated_at: r.updated_at,
|
|
1727
2217
|
tags: tagMap.get(r.id) ?? [],
|
|
1728
2218
|
has_embedding: true
|
|
1729
|
-
// Semantic results always have embeddings (JOIN on embedding table)
|
|
1730
2219
|
})),
|
|
1731
2220
|
total,
|
|
1732
2221
|
query: "",
|
|
@@ -1737,21 +2226,19 @@ var SearchRepository = class {
|
|
|
1737
2226
|
facets
|
|
1738
2227
|
};
|
|
1739
2228
|
}
|
|
1740
|
-
/**
|
|
1741
|
-
* Hybrid search combining BM25 (full-text) and vector similarity using
|
|
1742
|
-
* Reciprocal Rank Fusion (RRF, k=60).
|
|
1743
|
-
*/
|
|
1744
2229
|
async hybridSearch(query, queryEmbedding, options = {}) {
|
|
1745
2230
|
const { limit = 20, offset = 0 } = options;
|
|
1746
2231
|
const vectorStr = `[${queryEmbedding.join(",")}]`;
|
|
1747
2232
|
const k = 60;
|
|
1748
2233
|
const tsqFn = this.tsqueryFn(query);
|
|
2234
|
+
const resolvedEmbeddingSet = await this.resolveEmbeddingSet(options);
|
|
1749
2235
|
const textCond = buildNoteConditions(options, 2);
|
|
1750
2236
|
const textConditions = [
|
|
1751
2237
|
...textCond.conditions,
|
|
1752
2238
|
`(n.tsv @@ ${tsqFn}('english', $1) OR
|
|
1753
2239
|
to_tsvector('english', coalesce(c.content, '')) @@ ${tsqFn}('english', $1))`
|
|
1754
2240
|
];
|
|
2241
|
+
this.scopeToResolvedEmbeddingSet(textConditions, textCond.params, textCond.nextIdx, resolvedEmbeddingSet);
|
|
1755
2242
|
const textWhere = textConditions.join(" AND ");
|
|
1756
2243
|
const textParams = [query, ...textCond.params];
|
|
1757
2244
|
const textResult = await this.db.query(
|
|
@@ -1768,6 +2255,7 @@ var SearchRepository = class {
|
|
|
1768
2255
|
textParams
|
|
1769
2256
|
);
|
|
1770
2257
|
const vecCond = buildNoteConditions(options, 1);
|
|
2258
|
+
vecCond.nextIdx = this.scopeToResolvedEmbeddingRows(vecCond.conditions, vecCond.params, vecCond.nextIdx, resolvedEmbeddingSet);
|
|
1771
2259
|
const vecWhere = vecCond.conditions.join(" AND ");
|
|
1772
2260
|
const vecVecIdx = vecCond.nextIdx;
|
|
1773
2261
|
const vectorResult = await this.db.query(
|
|
@@ -1781,26 +2269,16 @@ var SearchRepository = class {
|
|
|
1781
2269
|
);
|
|
1782
2270
|
const rrfScores = /* @__PURE__ */ new Map();
|
|
1783
2271
|
textResult.rows.forEach((row, idx) => {
|
|
1784
|
-
|
|
1785
|
-
rrfScores.set(row.id, (rrfScores.get(row.id) ?? 0) + score);
|
|
2272
|
+
rrfScores.set(row.id, (rrfScores.get(row.id) ?? 0) + 1 / (k + idx + 1));
|
|
1786
2273
|
});
|
|
1787
2274
|
vectorResult.rows.forEach((row, idx) => {
|
|
1788
|
-
|
|
1789
|
-
rrfScores.set(row.id, (rrfScores.get(row.id) ?? 0) + score);
|
|
2275
|
+
rrfScores.set(row.id, (rrfScores.get(row.id) ?? 0) + 1 / (k + idx + 1));
|
|
1790
2276
|
});
|
|
1791
2277
|
const sortedIds = Array.from(rrfScores.entries()).sort((a, b) => b[1] - a[1]).map(([id]) => id);
|
|
1792
2278
|
const total = sortedIds.length;
|
|
1793
2279
|
const pageIds = sortedIds.slice(offset, offset + limit);
|
|
1794
2280
|
if (pageIds.length === 0) {
|
|
1795
|
-
return {
|
|
1796
|
-
results: [],
|
|
1797
|
-
total,
|
|
1798
|
-
query,
|
|
1799
|
-
mode: "hybrid",
|
|
1800
|
-
semantic_available: this.semanticAvailable,
|
|
1801
|
-
limit,
|
|
1802
|
-
offset
|
|
1803
|
-
};
|
|
2281
|
+
return { results: [], total, query, mode: "hybrid", semantic_available: this.semanticAvailable, limit, offset };
|
|
1804
2282
|
}
|
|
1805
2283
|
const noteResult = await this.db.query(
|
|
1806
2284
|
`SELECT n.id, n.title, n.created_at, n.updated_at,
|
|
@@ -1813,7 +2291,7 @@ var SearchRepository = class {
|
|
|
1813
2291
|
const noteMap = new Map(noteResult.rows.map((r) => [r.id, r]));
|
|
1814
2292
|
const [tagMap, embeddingSet] = await Promise.all([
|
|
1815
2293
|
this.fetchTagMap(pageIds),
|
|
1816
|
-
this.
|
|
2294
|
+
this.fetchEmbeddingStatus(pageIds, resolvedEmbeddingSet, options.embeddingSetId)
|
|
1817
2295
|
]);
|
|
1818
2296
|
const facets = options.include_facets ? await this.fetchFacets(sortedIds) : void 0;
|
|
1819
2297
|
return {
|
|
@@ -1842,8 +2320,9 @@ var SearchRepository = class {
|
|
|
1842
2320
|
}
|
|
1843
2321
|
async recentNotes(options = {}) {
|
|
1844
2322
|
const { limit = 20, offset = 0 } = options;
|
|
2323
|
+
const resolvedEmbeddingSet = await this.resolveEmbeddingSet(options);
|
|
1845
2324
|
const { conditions, params, nextIdx } = buildNoteConditions(options, 1);
|
|
1846
|
-
let paramIdx = nextIdx;
|
|
2325
|
+
let paramIdx = this.scopeToResolvedEmbeddingSet(conditions, params, nextIdx, resolvedEmbeddingSet);
|
|
1847
2326
|
const where = conditions.join(" AND ");
|
|
1848
2327
|
const countResult = await this.db.query(
|
|
1849
2328
|
`SELECT COUNT(*) as count FROM note n WHERE ${where}`,
|
|
@@ -1862,7 +2341,7 @@ var SearchRepository = class {
|
|
|
1862
2341
|
listParams
|
|
1863
2342
|
);
|
|
1864
2343
|
const resultIds = result.rows.map((r) => r.id);
|
|
1865
|
-
const embeddingSet = await this.
|
|
2344
|
+
const embeddingSet = await this.fetchEmbeddingStatus(resultIds, resolvedEmbeddingSet, options.embeddingSetId);
|
|
1866
2345
|
return {
|
|
1867
2346
|
results: result.rows.map((r) => ({
|
|
1868
2347
|
id: r.id,
|
|
@@ -1882,14 +2361,8 @@ var SearchRepository = class {
|
|
|
1882
2361
|
offset
|
|
1883
2362
|
};
|
|
1884
2363
|
}
|
|
1885
|
-
/**
|
|
1886
|
-
* Fetch faceted aggregate counts for tags and collections across all matching note IDs.
|
|
1887
|
-
* Uses the full (unpaginated) result set for accurate counts.
|
|
1888
|
-
*/
|
|
1889
2364
|
async fetchFacets(noteIds) {
|
|
1890
|
-
if (noteIds.length === 0) {
|
|
1891
|
-
return { tags: [], collections: [] };
|
|
1892
|
-
}
|
|
2365
|
+
if (noteIds.length === 0) return { tags: [], collections: [] };
|
|
1893
2366
|
const [tagResult, collResult] = await Promise.all([
|
|
1894
2367
|
this.db.query(
|
|
1895
2368
|
`SELECT nt.tag, COUNT(*) as count FROM note_tag nt
|
|
@@ -1924,6 +2397,430 @@ var SearchRepository = class {
|
|
|
1924
2397
|
}
|
|
1925
2398
|
};
|
|
1926
2399
|
|
|
2400
|
+
// src/repositories/graph-repository.ts
|
|
2401
|
+
var SIMILARITY_GRAPH_ALGORITHM = "knn-v1";
|
|
2402
|
+
function hashJson(value) {
|
|
2403
|
+
return computeHash(new TextEncoder().encode(JSON.stringify(value)));
|
|
2404
|
+
}
|
|
2405
|
+
function sourceIdFor(inputHash) {
|
|
2406
|
+
return `similarity-${inputHash.replace(/^sha256:/, "").slice(0, 24)}`;
|
|
2407
|
+
}
|
|
2408
|
+
function jsonObject(value) {
|
|
2409
|
+
if (value == null) return null;
|
|
2410
|
+
if (typeof value === "string") return JSON.parse(value);
|
|
2411
|
+
return value;
|
|
2412
|
+
}
|
|
2413
|
+
function detectCommunities(edges, nodes = [], options = {}) {
|
|
2414
|
+
const nodeIds = new Set(nodes.map((n) => n.id));
|
|
2415
|
+
for (const edge of edges) {
|
|
2416
|
+
nodeIds.add(edge.source);
|
|
2417
|
+
nodeIds.add(edge.target);
|
|
2418
|
+
}
|
|
2419
|
+
const labels = /* @__PURE__ */ new Map();
|
|
2420
|
+
const adjacency = /* @__PURE__ */ new Map();
|
|
2421
|
+
for (const id of nodeIds) {
|
|
2422
|
+
labels.set(id, id);
|
|
2423
|
+
adjacency.set(id, []);
|
|
2424
|
+
}
|
|
2425
|
+
for (const edge of edges) {
|
|
2426
|
+
adjacency.get(edge.source)?.push({ node: edge.target, weight: edge.weight });
|
|
2427
|
+
adjacency.get(edge.target)?.push({ node: edge.source, weight: edge.weight });
|
|
2428
|
+
}
|
|
2429
|
+
const orderedNodes = Array.from(nodeIds).sort();
|
|
2430
|
+
const maxIterations = options.maxIterations ?? 20;
|
|
2431
|
+
for (let i = 0; i < maxIterations; i++) {
|
|
2432
|
+
let changed = false;
|
|
2433
|
+
for (const node of orderedNodes) {
|
|
2434
|
+
const scores = /* @__PURE__ */ new Map();
|
|
2435
|
+
for (const neighbor of adjacency.get(node) ?? []) {
|
|
2436
|
+
const label = labels.get(neighbor.node) ?? neighbor.node;
|
|
2437
|
+
scores.set(label, (scores.get(label) ?? 0) + neighbor.weight);
|
|
2438
|
+
}
|
|
2439
|
+
if (scores.size === 0) continue;
|
|
2440
|
+
const nextLabel = Array.from(scores.entries()).sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))[0][0];
|
|
2441
|
+
if (nextLabel !== labels.get(node)) {
|
|
2442
|
+
labels.set(node, nextLabel);
|
|
2443
|
+
changed = true;
|
|
2444
|
+
}
|
|
2445
|
+
}
|
|
2446
|
+
if (!changed) break;
|
|
2447
|
+
}
|
|
2448
|
+
const byLabel = /* @__PURE__ */ new Map();
|
|
2449
|
+
for (const node of orderedNodes) {
|
|
2450
|
+
const label = labels.get(node) ?? node;
|
|
2451
|
+
const members = byLabel.get(label) ?? [];
|
|
2452
|
+
members.push(node);
|
|
2453
|
+
byLabel.set(label, members);
|
|
2454
|
+
}
|
|
2455
|
+
return Array.from(byLabel.values()).sort((a, b) => b.length - a.length || a[0].localeCompare(b[0])).map((members, index) => ({ id: `community-${index + 1}`, nodes: members }));
|
|
2456
|
+
}
|
|
2457
|
+
var GraphRepository = class {
|
|
2458
|
+
constructor(db) {
|
|
2459
|
+
this.db = db;
|
|
2460
|
+
}
|
|
2461
|
+
normalizeSimilarityRequest(request) {
|
|
2462
|
+
if (request.minSimilarity !== void 0 && request.threshold !== void 0 && request.minSimilarity !== request.threshold) {
|
|
2463
|
+
throw new Error("conflicting-threshold");
|
|
2464
|
+
}
|
|
2465
|
+
return {
|
|
2466
|
+
selector: request.selector,
|
|
2467
|
+
k: request.k ?? 5,
|
|
2468
|
+
minSimilarity: request.minSimilarity ?? request.threshold ?? -1,
|
|
2469
|
+
metric: request.metric ?? "cosine",
|
|
2470
|
+
source: request.source ?? "cache-preferred"
|
|
2471
|
+
};
|
|
2472
|
+
}
|
|
2473
|
+
async buildSimilarityGraph(embeddingSet, options = {}) {
|
|
2474
|
+
const selector = typeof embeddingSet === "string" ? { kind: "embedding-set", embeddingSetId: embeddingSet } : embeddingSet;
|
|
2475
|
+
return this.buildSimilarityGraphFromResolved(
|
|
2476
|
+
await new EmbeddingSetsRepository(this.db).resolveSelector(selector),
|
|
2477
|
+
options
|
|
2478
|
+
);
|
|
2479
|
+
}
|
|
2480
|
+
async buildSimilarityGraphLive(request) {
|
|
2481
|
+
const normalized = this.normalizeSimilarityRequest({ ...request, source: "live-only" });
|
|
2482
|
+
const resolved = await new EmbeddingSetsRepository(this.db).resolveSelector(normalized.selector);
|
|
2483
|
+
return this.buildSimilarityGraphFromResolved(resolved, normalized);
|
|
2484
|
+
}
|
|
2485
|
+
async getCachedSimilarityGraph(request) {
|
|
2486
|
+
const normalized = this.normalizeSimilarityRequest(request);
|
|
2487
|
+
const resolved = await new EmbeddingSetsRepository(this.db).resolveSelector(normalized.selector);
|
|
2488
|
+
const cacheKey = await this.computeSimilarityGraphCacheKey(normalized, resolved);
|
|
2489
|
+
const inputHash = hashJson(cacheKey);
|
|
2490
|
+
const source = await this.findGraphSource(inputHash);
|
|
2491
|
+
if (!source) return null;
|
|
2492
|
+
const graph = await this.graphFromArtifact(source.id, resolved.noteIds);
|
|
2493
|
+
return {
|
|
2494
|
+
graph,
|
|
2495
|
+
graphSource: { id: source.id, name: source.name, input_hash: source.input_hash, freshness: source.freshness },
|
|
2496
|
+
cache: "hit",
|
|
2497
|
+
freshness: source.freshness
|
|
2498
|
+
};
|
|
2499
|
+
}
|
|
2500
|
+
async buildOrLoadSimilarityGraph(request) {
|
|
2501
|
+
const normalized = this.normalizeSimilarityRequest(request);
|
|
2502
|
+
const resolved = await new EmbeddingSetsRepository(this.db).resolveSelector(normalized.selector);
|
|
2503
|
+
const cacheKey = await this.computeSimilarityGraphCacheKey(normalized, resolved);
|
|
2504
|
+
const inputHash = hashJson(cacheKey);
|
|
2505
|
+
if (normalized.source !== "live-only") {
|
|
2506
|
+
const cached = await this.findGraphSource(inputHash);
|
|
2507
|
+
if (cached?.freshness === "fresh") {
|
|
2508
|
+
const graph3 = await this.graphFromArtifact(cached.id, resolved.noteIds);
|
|
2509
|
+
return {
|
|
2510
|
+
graph: graph3,
|
|
2511
|
+
graphSource: { id: cached.id, name: cached.name, input_hash: cached.input_hash, freshness: cached.freshness },
|
|
2512
|
+
cache: "hit",
|
|
2513
|
+
freshness: "fresh"
|
|
2514
|
+
};
|
|
2515
|
+
}
|
|
2516
|
+
if (normalized.source === "cache-only") {
|
|
2517
|
+
throw new Error(cached ? "similarity graph cache stale" : "similarity graph cache miss");
|
|
2518
|
+
}
|
|
2519
|
+
const graph2 = await this.buildSimilarityGraphFromResolved(resolved, normalized);
|
|
2520
|
+
const graphSource = await this.saveSimilarityGraphArtifact({ graph: graph2, request: normalized, resolved, cacheKey, freshness: "fresh" });
|
|
2521
|
+
return {
|
|
2522
|
+
graph: graph2,
|
|
2523
|
+
graphSource,
|
|
2524
|
+
cache: cached ? "stale-live-built" : "miss-live-built",
|
|
2525
|
+
freshness: cached ? cached.freshness : "fresh"
|
|
2526
|
+
};
|
|
2527
|
+
}
|
|
2528
|
+
const graph = await this.buildSimilarityGraphFromResolved(resolved, normalized);
|
|
2529
|
+
return {
|
|
2530
|
+
graph,
|
|
2531
|
+
graphSource: { id: sourceIdFor(inputHash), name: "Live similarity graph", input_hash: inputHash, freshness: "unknown" },
|
|
2532
|
+
cache: "live-only",
|
|
2533
|
+
freshness: "unknown"
|
|
2534
|
+
};
|
|
2535
|
+
}
|
|
2536
|
+
async saveSimilarityGraphArtifact(input) {
|
|
2537
|
+
const inputHash = hashJson(input.cacheKey);
|
|
2538
|
+
const id = sourceIdFor(inputHash);
|
|
2539
|
+
const parameters = {
|
|
2540
|
+
k: input.request.k,
|
|
2541
|
+
minSimilarity: input.request.minSimilarity,
|
|
2542
|
+
metric: input.request.metric,
|
|
2543
|
+
algorithm: SIMILARITY_GRAPH_ALGORITHM,
|
|
2544
|
+
selectorHash: input.cacheKey.selectorHash,
|
|
2545
|
+
parameterHash: input.cacheKey.parameterHash
|
|
2546
|
+
};
|
|
2547
|
+
await this.db.query(
|
|
2548
|
+
`INSERT INTO graph_source (id, name, kind, source_table, embedding_set_id, virtual_set_id, model, dimension, metric, algorithm, parameters_json, input_hash, freshness_json)
|
|
2549
|
+
VALUES ($1, $2, 'similarity', 'embedding', $3, $4, $5, $6, $7, $8, $9::jsonb, $10, $11::jsonb)
|
|
2550
|
+
ON CONFLICT (id) DO UPDATE SET parameters_json = $9::jsonb, input_hash = $10, freshness_json = $11::jsonb`,
|
|
2551
|
+
[
|
|
2552
|
+
id,
|
|
2553
|
+
"Cached similarity graph",
|
|
2554
|
+
input.request.selector.kind === "embedding-set" ? input.request.selector.embeddingSetId ?? null : null,
|
|
2555
|
+
input.request.selector.kind !== "embedding-set" ? input.request.selector.definition?.id ?? null : null,
|
|
2556
|
+
input.cacheKey.model,
|
|
2557
|
+
input.cacheKey.dimension,
|
|
2558
|
+
input.request.metric,
|
|
2559
|
+
SIMILARITY_GRAPH_ALGORITHM,
|
|
2560
|
+
JSON.stringify(parameters),
|
|
2561
|
+
inputHash,
|
|
2562
|
+
JSON.stringify({ status: input.freshness ?? "fresh" })
|
|
2563
|
+
]
|
|
2564
|
+
);
|
|
2565
|
+
await this.db.query(`DELETE FROM graph_edge_artifact WHERE graph_source_id = $1`, [id]);
|
|
2566
|
+
let rank = 1;
|
|
2567
|
+
for (const edge of input.graph.edges) {
|
|
2568
|
+
await this.db.query(
|
|
2569
|
+
`INSERT INTO graph_edge_artifact (graph_source_id, from_note_id, to_note_id, weight, kind, rank)
|
|
2570
|
+
VALUES ($1, $2, $3, $4, 'similarity', $5)`,
|
|
2571
|
+
[id, edge.source, edge.target, edge.weight, rank++]
|
|
2572
|
+
);
|
|
2573
|
+
}
|
|
2574
|
+
return { id, name: "Cached similarity graph", input_hash: inputHash, freshness: input.freshness ?? "fresh" };
|
|
2575
|
+
}
|
|
2576
|
+
async markSimilarityGraphStale(graphSourceId, reason) {
|
|
2577
|
+
await this.db.query(
|
|
2578
|
+
`UPDATE graph_source SET freshness_json = $2::jsonb WHERE id = $1`,
|
|
2579
|
+
[graphSourceId, JSON.stringify({ status: "stale", stale_reason: reason, checked_at: (/* @__PURE__ */ new Date()).toISOString() })]
|
|
2580
|
+
);
|
|
2581
|
+
}
|
|
2582
|
+
async loadGraphArtifact(graphSourceId, noteIds = []) {
|
|
2583
|
+
return this.graphFromArtifact(graphSourceId, noteIds);
|
|
2584
|
+
}
|
|
2585
|
+
async buildSimilarityGraphFromResolved(resolved, options) {
|
|
2586
|
+
const k = options.k ?? 5;
|
|
2587
|
+
const minSimilarity = options.minSimilarity ?? options.threshold ?? -1;
|
|
2588
|
+
const embeddings = resolved.rows;
|
|
2589
|
+
const nodes = embeddings.map((row) => ({ id: row.note_id }));
|
|
2590
|
+
const edgeMap = /* @__PURE__ */ new Map();
|
|
2591
|
+
for (const row of embeddings) {
|
|
2592
|
+
const neighbors = await this.db.query(
|
|
2593
|
+
`SELECT note_id, 1 - (vector <=> $2::vector) as similarity
|
|
2594
|
+
FROM embedding
|
|
2595
|
+
WHERE id = ANY($1) AND note_id != $3
|
|
2596
|
+
ORDER BY vector <=> $2::vector ASC
|
|
2597
|
+
LIMIT $4`,
|
|
2598
|
+
[resolved.embeddingIds, row.vector, row.note_id, k]
|
|
2599
|
+
);
|
|
2600
|
+
for (const neighbor of neighbors.rows) {
|
|
2601
|
+
if (neighbor.similarity < minSimilarity) continue;
|
|
2602
|
+
const [source, target] = [row.note_id, neighbor.note_id].sort();
|
|
2603
|
+
const id = `${source}\0${target}`;
|
|
2604
|
+
const existing = edgeMap.get(id);
|
|
2605
|
+
if (!existing || neighbor.similarity > existing.weight) {
|
|
2606
|
+
edgeMap.set(id, { source, target, weight: neighbor.similarity, kind: "similarity" });
|
|
2607
|
+
}
|
|
2608
|
+
}
|
|
2609
|
+
}
|
|
2610
|
+
const edges = Array.from(edgeMap.values()).sort(
|
|
2611
|
+
(a, b) => a.source.localeCompare(b.source) || a.target.localeCompare(b.target)
|
|
2612
|
+
);
|
|
2613
|
+
return { nodes, edges, communities: detectCommunities(edges, nodes) };
|
|
2614
|
+
}
|
|
2615
|
+
async computeSimilarityGraphCacheKey(request, resolved) {
|
|
2616
|
+
const firstSetId = resolved.rows[0]?.embedding_set_id;
|
|
2617
|
+
const set = firstSetId ? await new EmbeddingSetsRepository(this.db).get(firstSetId) : null;
|
|
2618
|
+
return {
|
|
2619
|
+
selectorHash: hashJson(request.selector),
|
|
2620
|
+
resolvedEmbeddingSetId: request.selector.kind === "embedding-set" ? request.selector.embeddingSetId : void 0,
|
|
2621
|
+
virtualSetId: request.selector.kind === "virtual-definition" ? request.selector.definition?.id : void 0,
|
|
2622
|
+
k: request.k,
|
|
2623
|
+
minSimilarity: request.minSimilarity,
|
|
2624
|
+
metric: request.metric,
|
|
2625
|
+
model: set?.model_name ?? "unknown",
|
|
2626
|
+
dimension: set?.dimensions ?? 0,
|
|
2627
|
+
truncateDimension: set?.truncate_dimension ?? null,
|
|
2628
|
+
memberHash: hashJson(resolved.rows.map((row) => [row.note_id, row.embedding_set_id, row.embedding_id])),
|
|
2629
|
+
vectorHash: hashJson(resolved.rows.map((row) => [row.embedding_id, row.vector])),
|
|
2630
|
+
parameterHash: hashJson({ k: request.k, minSimilarity: request.minSimilarity, metric: request.metric, algorithm: SIMILARITY_GRAPH_ALGORITHM })
|
|
2631
|
+
};
|
|
2632
|
+
}
|
|
2633
|
+
async findGraphSource(inputHash) {
|
|
2634
|
+
const result = await this.db.query(
|
|
2635
|
+
`SELECT id, name, input_hash, freshness_json FROM graph_source WHERE kind = 'similarity' AND input_hash = $1 LIMIT 1`,
|
|
2636
|
+
[inputHash]
|
|
2637
|
+
);
|
|
2638
|
+
if (result.rows.length === 0) return null;
|
|
2639
|
+
const row = result.rows[0];
|
|
2640
|
+
const freshness = jsonObject(row.freshness_json);
|
|
2641
|
+
return { id: row.id, name: row.name, input_hash: row.input_hash, freshness: freshness?.status ?? "unknown" };
|
|
2642
|
+
}
|
|
2643
|
+
async graphFromArtifact(graphSourceId, noteIds) {
|
|
2644
|
+
const result = await this.db.query(
|
|
2645
|
+
`SELECT from_note_id as source, to_note_id as target, weight, kind
|
|
2646
|
+
FROM graph_edge_artifact
|
|
2647
|
+
WHERE graph_source_id = $1
|
|
2648
|
+
ORDER BY from_note_id, to_note_id`,
|
|
2649
|
+
[graphSourceId]
|
|
2650
|
+
);
|
|
2651
|
+
const nodes = Array.from(/* @__PURE__ */ new Set([...noteIds, ...result.rows.flatMap((edge) => [edge.source, edge.target])])).sort().map((id) => ({ id }));
|
|
2652
|
+
return { nodes, edges: result.rows, communities: detectCommunities(result.rows, nodes) };
|
|
2653
|
+
}
|
|
2654
|
+
async buildLinkGraph(linkType) {
|
|
2655
|
+
const params = [];
|
|
2656
|
+
const typeFilter = linkType ? " AND link_type = $1" : "";
|
|
2657
|
+
if (linkType) params.push(linkType);
|
|
2658
|
+
const result = await this.db.query(
|
|
2659
|
+
`SELECT source_note_id as source, target_note_id as target,
|
|
2660
|
+
COALESCE(confidence, 1.0) as weight, link_type as kind
|
|
2661
|
+
FROM link
|
|
2662
|
+
WHERE deleted_at IS NULL${typeFilter}
|
|
2663
|
+
ORDER BY source_note_id, target_note_id`,
|
|
2664
|
+
params
|
|
2665
|
+
);
|
|
2666
|
+
const nodes = Array.from(new Set(result.rows.flatMap((edge) => [edge.source, edge.target]))).sort().map((id) => ({ id }));
|
|
2667
|
+
return { nodes, edges: result.rows, communities: detectCommunities(result.rows, nodes) };
|
|
2668
|
+
}
|
|
2669
|
+
};
|
|
2670
|
+
|
|
2671
|
+
// src/repositories/communities-repository.ts
|
|
2672
|
+
function json(value) {
|
|
2673
|
+
return value == null ? null : JSON.stringify(value);
|
|
2674
|
+
}
|
|
2675
|
+
function parseObject(value) {
|
|
2676
|
+
if (value == null) return void 0;
|
|
2677
|
+
if (typeof value === "string") return JSON.parse(value);
|
|
2678
|
+
return value;
|
|
2679
|
+
}
|
|
2680
|
+
function iso(value) {
|
|
2681
|
+
if (!value) return void 0;
|
|
2682
|
+
return value instanceof Date ? value.toISOString() : value;
|
|
2683
|
+
}
|
|
2684
|
+
var CommunitiesRepository = class {
|
|
2685
|
+
constructor(db) {
|
|
2686
|
+
this.db = db;
|
|
2687
|
+
}
|
|
2688
|
+
async previewDynamicCommunity(filters) {
|
|
2689
|
+
const noteIds = await this.resolveFilterNoteIds(filters);
|
|
2690
|
+
return noteIds.map((noteId) => ({
|
|
2691
|
+
communitySourceId: "dynamic-preview",
|
|
2692
|
+
communityId: "dynamic-preview",
|
|
2693
|
+
noteId,
|
|
2694
|
+
label: "Dynamic preview",
|
|
2695
|
+
confidence: null,
|
|
2696
|
+
sourceType: "dynamic"
|
|
2697
|
+
}));
|
|
2698
|
+
}
|
|
2699
|
+
async saveCommunity(input) {
|
|
2700
|
+
const sourceId = generateId();
|
|
2701
|
+
const communitySetId = generateId();
|
|
2702
|
+
const communityId = generateId();
|
|
2703
|
+
const noteIds = input.noteIds ?? (input.filters ? await this.resolveFilterNoteIds(input.filters) : []);
|
|
2704
|
+
const sourceKind = input.sourceType === "dynamic-snapshot" ? "search" : "manual";
|
|
2705
|
+
const freshness = input.sourceType === "dynamic-snapshot" ? "fresh" : "unknown";
|
|
2706
|
+
await this.db.query(
|
|
2707
|
+
`INSERT INTO graph_source (id, name, kind, source_table, parameters_json, input_hash, freshness_json)
|
|
2708
|
+
VALUES ($1, $2, $3, 'manual', $4::jsonb, $5, $6::jsonb)`,
|
|
2709
|
+
[sourceId, input.name, sourceKind, json({ filters: input.filters ?? null }), `community:${sourceId}`, json({ status: freshness })]
|
|
2710
|
+
);
|
|
2711
|
+
await this.db.query(
|
|
2712
|
+
`INSERT INTO community_set (id, graph_source_id, name, source_type, parameters_json, input_hash, freshness_json)
|
|
2713
|
+
VALUES ($1, $2, $3, $4, $5::jsonb, $6, $7::jsonb)`,
|
|
2714
|
+
[communitySetId, sourceId, input.name, input.sourceType, json({ filters: input.filters ?? null }), `community:${communitySetId}`, json({ status: freshness })]
|
|
2715
|
+
);
|
|
2716
|
+
await this.db.query(
|
|
2717
|
+
`INSERT INTO community (community_set_id, id, label, rank, size, representative_note_ids)
|
|
2718
|
+
VALUES ($1, $2, $3, 1, $4, $5)`,
|
|
2719
|
+
[communitySetId, communityId, input.label ?? input.name, noteIds.length, input.representativeNoteIds ?? []]
|
|
2720
|
+
);
|
|
2721
|
+
for (const noteId of noteIds) {
|
|
2722
|
+
await this.db.query(
|
|
2723
|
+
`INSERT INTO community_assignment (community_set_id, community_id, note_id, confidence, source_type)
|
|
2724
|
+
VALUES ($1, $2, $3, NULL, $4)`,
|
|
2725
|
+
[communitySetId, communityId, noteId, input.sourceType]
|
|
2726
|
+
);
|
|
2727
|
+
}
|
|
2728
|
+
return {
|
|
2729
|
+
id: communitySetId,
|
|
2730
|
+
name: input.name,
|
|
2731
|
+
sourceType: input.sourceType,
|
|
2732
|
+
graphSourceId: sourceId,
|
|
2733
|
+
searchQuery: input.filters?.query,
|
|
2734
|
+
filters: input.filters,
|
|
2735
|
+
freshness
|
|
2736
|
+
};
|
|
2737
|
+
}
|
|
2738
|
+
async rerunDynamicCommunity(sourceId) {
|
|
2739
|
+
const result = await this.db.query(
|
|
2740
|
+
`SELECT parameters_json, source_type FROM community_set WHERE id = $1`,
|
|
2741
|
+
[sourceId]
|
|
2742
|
+
);
|
|
2743
|
+
if (result.rows.length === 0) throw new Error(`Community source not found: ${sourceId}`);
|
|
2744
|
+
const parameters = parseObject(result.rows[0].parameters_json);
|
|
2745
|
+
if (!parameters?.filters) return this.getCommunityAssignments(sourceId);
|
|
2746
|
+
const preview = await this.previewDynamicCommunity(parameters.filters);
|
|
2747
|
+
return preview.map((assignment) => ({ ...assignment, communitySourceId: sourceId, sourceType: "dynamic" }));
|
|
2748
|
+
}
|
|
2749
|
+
async listCommunitySources() {
|
|
2750
|
+
const result = await this.db.query(`SELECT * FROM community_set ORDER BY created_at, name`);
|
|
2751
|
+
return result.rows.map((row) => {
|
|
2752
|
+
const parameters = parseObject(row.parameters_json);
|
|
2753
|
+
const freshness = parseObject(row.freshness_json);
|
|
2754
|
+
return {
|
|
2755
|
+
id: row.id,
|
|
2756
|
+
name: row.name,
|
|
2757
|
+
sourceType: row.source_type,
|
|
2758
|
+
graphSourceId: row.graph_source_id,
|
|
2759
|
+
searchQuery: parameters?.filters?.query,
|
|
2760
|
+
filters: parameters?.filters ?? void 0,
|
|
2761
|
+
createdAt: iso(row.created_at),
|
|
2762
|
+
freshness: freshness?.status ?? "unknown"
|
|
2763
|
+
};
|
|
2764
|
+
});
|
|
2765
|
+
}
|
|
2766
|
+
async getCommunityAssignments(sourceId) {
|
|
2767
|
+
const result = await this.db.query(
|
|
2768
|
+
`SELECT ca.community_set_id, ca.community_id, ca.note_id, c.label, ca.confidence, ca.source_type
|
|
2769
|
+
FROM community_assignment ca
|
|
2770
|
+
LEFT JOIN community c ON c.community_set_id = ca.community_set_id AND c.id = ca.community_id
|
|
2771
|
+
WHERE ca.community_set_id = $1
|
|
2772
|
+
ORDER BY ca.community_id, ca.note_id`,
|
|
2773
|
+
[sourceId]
|
|
2774
|
+
);
|
|
2775
|
+
return result.rows.map((row) => ({
|
|
2776
|
+
communitySourceId: row.community_set_id,
|
|
2777
|
+
communityId: row.community_id,
|
|
2778
|
+
noteId: row.note_id,
|
|
2779
|
+
label: row.label,
|
|
2780
|
+
confidence: row.confidence,
|
|
2781
|
+
sourceType: row.source_type
|
|
2782
|
+
}));
|
|
2783
|
+
}
|
|
2784
|
+
async listCommunitySummaries(sourceId) {
|
|
2785
|
+
const result = await this.db.query(
|
|
2786
|
+
`SELECT c.id, c.label, cs.source_type, c.size, c.confidence, c.representative_note_ids, cs.freshness_json
|
|
2787
|
+
FROM community c
|
|
2788
|
+
JOIN community_set cs ON cs.id = c.community_set_id
|
|
2789
|
+
WHERE c.community_set_id = $1
|
|
2790
|
+
ORDER BY c.rank NULLS LAST, c.id`,
|
|
2791
|
+
[sourceId]
|
|
2792
|
+
);
|
|
2793
|
+
return result.rows.map((row) => {
|
|
2794
|
+
const freshness = parseObject(row.freshness_json);
|
|
2795
|
+
return {
|
|
2796
|
+
id: row.id,
|
|
2797
|
+
label: row.label ?? row.id,
|
|
2798
|
+
sourceType: row.source_type,
|
|
2799
|
+
size: row.size ?? 0,
|
|
2800
|
+
confidence: row.confidence,
|
|
2801
|
+
representativeNoteIds: row.representative_note_ids ?? [],
|
|
2802
|
+
freshness: freshness?.status ?? "unknown"
|
|
2803
|
+
};
|
|
2804
|
+
});
|
|
2805
|
+
}
|
|
2806
|
+
async resolveFilterNoteIds(filters) {
|
|
2807
|
+
if (filters.conceptIds?.length) {
|
|
2808
|
+
throw new Error("Community concept filters are not supported locally yet");
|
|
2809
|
+
}
|
|
2810
|
+
if (filters.noteIds?.length && !filters.query && !filters.tags?.length && !filters.collectionIds?.length && !filters.embeddingSetSelector) {
|
|
2811
|
+
return [...filters.noteIds].sort();
|
|
2812
|
+
}
|
|
2813
|
+
const result = await new SearchRepository(this.db, true).search(filters.query ?? "", {
|
|
2814
|
+
limit: 1e3,
|
|
2815
|
+
tags: filters.tags,
|
|
2816
|
+
collection_id: filters.collectionIds?.[0],
|
|
2817
|
+
embeddingSetSelector: filters.embeddingSetSelector
|
|
2818
|
+
});
|
|
2819
|
+
const ids = result.results.map((row) => row.id);
|
|
2820
|
+
return filters.noteIds?.length ? ids.filter((id) => filters.noteIds?.includes(id)) : ids;
|
|
2821
|
+
}
|
|
2822
|
+
};
|
|
2823
|
+
|
|
1927
2824
|
// src/capabilities/llm-handler.ts
|
|
1928
2825
|
var llmFn = null;
|
|
1929
2826
|
function setLlmFunction(fn) {
|
|
@@ -2056,12 +2953,12 @@ var JobQueueWorker = class {
|
|
|
2056
2953
|
if (job.required_capability) {
|
|
2057
2954
|
const capName = job.required_capability;
|
|
2058
2955
|
if (!this.capabilityManager?.isReady(capName)) {
|
|
2059
|
-
|
|
2956
|
+
await this.blockForCapability(job, capName);
|
|
2060
2957
|
continue;
|
|
2061
2958
|
}
|
|
2062
2959
|
}
|
|
2063
2960
|
await this.db.query(
|
|
2064
|
-
`UPDATE job_queue SET status = 'processing', updated_at = now() WHERE id = $1`,
|
|
2961
|
+
`UPDATE job_queue SET status = 'processing', error = NULL, updated_at = now() WHERE id = $1`,
|
|
2065
2962
|
[job.id]
|
|
2066
2963
|
);
|
|
2067
2964
|
try {
|
|
@@ -2069,7 +2966,7 @@ var JobQueueWorker = class {
|
|
|
2069
2966
|
const jobResult = await handler(job, this.db);
|
|
2070
2967
|
console.log(`[JobQueue] Completed ${job.job_type}:`, jobResult);
|
|
2071
2968
|
await this.db.query(
|
|
2072
|
-
`UPDATE job_queue SET status = 'completed', result = $1, updated_at = now() WHERE id = $2`,
|
|
2969
|
+
`UPDATE job_queue SET status = 'completed', error = NULL, result = $1, updated_at = now() WHERE id = $2`,
|
|
2073
2970
|
[JSON.stringify(jobResult ?? null), job.id]
|
|
2074
2971
|
);
|
|
2075
2972
|
this.events?.emit("job.completed", {
|
|
@@ -2103,6 +3000,30 @@ var JobQueueWorker = class {
|
|
|
2103
3000
|
}
|
|
2104
3001
|
return processed;
|
|
2105
3002
|
}
|
|
3003
|
+
async blockForCapability(job, capability) {
|
|
3004
|
+
const message = `requires capability '${capability}' \u2014 not ready`;
|
|
3005
|
+
console.log(`[JobQueue] Deferring ${job.job_type} \u2014 ${message}`);
|
|
3006
|
+
await this.db.query(
|
|
3007
|
+
`UPDATE job_queue
|
|
3008
|
+
SET status = 'pending', error = $1, updated_at = now()
|
|
3009
|
+
WHERE id = $2 AND error IS DISTINCT FROM $1`,
|
|
3010
|
+
[message, job.id]
|
|
3011
|
+
);
|
|
3012
|
+
this.events?.emit("job.blocked", {
|
|
3013
|
+
id: job.id,
|
|
3014
|
+
noteId: job.note_id,
|
|
3015
|
+
type: job.job_type,
|
|
3016
|
+
capability,
|
|
3017
|
+
message
|
|
3018
|
+
});
|
|
3019
|
+
this.events?.emit("capability.required", {
|
|
3020
|
+
name: capability,
|
|
3021
|
+
jobId: job.id,
|
|
3022
|
+
noteId: job.note_id,
|
|
3023
|
+
type: job.job_type,
|
|
3024
|
+
message
|
|
3025
|
+
});
|
|
3026
|
+
}
|
|
2106
3027
|
getBackoffDelay(retryCount) {
|
|
2107
3028
|
const delay = this.options.backoffBaseMs * Math.pow(2, retryCount);
|
|
2108
3029
|
return Math.min(delay, this.options.backoffMaxMs);
|
|
@@ -3475,19 +4396,6 @@ function setEmbedFunction(fn) {
|
|
|
3475
4396
|
function getEmbedFunction() {
|
|
3476
4397
|
return embedFn;
|
|
3477
4398
|
}
|
|
3478
|
-
async function ensureEmbeddingSet(db) {
|
|
3479
|
-
const result = await db.query(
|
|
3480
|
-
`SELECT id FROM embedding_set WHERE model_name = $1`,
|
|
3481
|
-
["all-MiniLM-L6-v2"]
|
|
3482
|
-
);
|
|
3483
|
-
if (result.rows.length > 0) return result.rows[0].id;
|
|
3484
|
-
const id = generateId();
|
|
3485
|
-
await db.query(
|
|
3486
|
-
`INSERT INTO embedding_set (id, model_name, dimensions) VALUES ($1, $2, $3)`,
|
|
3487
|
-
[id, "all-MiniLM-L6-v2", 384]
|
|
3488
|
-
);
|
|
3489
|
-
return id;
|
|
3490
|
-
}
|
|
3491
4399
|
function averageEmbeddings(embeddings) {
|
|
3492
4400
|
if (embeddings.length === 1) return embeddings[0];
|
|
3493
4401
|
const dims = embeddings[0].length;
|
|
@@ -3515,27 +4423,14 @@ async function embeddingGenerationHandler(job, db) {
|
|
|
3515
4423
|
const chunks = chunkText(content);
|
|
3516
4424
|
const embeddings = await fn(chunks);
|
|
3517
4425
|
const vector2 = averageEmbeddings(embeddings);
|
|
3518
|
-
const
|
|
3519
|
-
await
|
|
3520
|
-
|
|
3521
|
-
|
|
3522
|
-
|
|
3523
|
-
|
|
3524
|
-
|
|
3525
|
-
|
|
3526
|
-
);
|
|
3527
|
-
const embId = generateId();
|
|
3528
|
-
const vectorStr = `[${vector2.join(",")}]`;
|
|
3529
|
-
await db.query(
|
|
3530
|
-
`INSERT INTO embedding (id, note_id, embedding_set_id, vector) VALUES ($1, $2, $3, $4::vector)`,
|
|
3531
|
-
[embId, job.note_id, setId, vectorStr]
|
|
3532
|
-
);
|
|
3533
|
-
await db.query(
|
|
3534
|
-
`INSERT INTO embedding_set_member (embedding_set_id, note_id, embedding_id) VALUES ($1, $2, $3)
|
|
3535
|
-
ON CONFLICT (embedding_set_id, note_id) DO UPDATE SET embedding_id = $3`,
|
|
3536
|
-
[setId, job.note_id, embId]
|
|
3537
|
-
);
|
|
3538
|
-
return { chunks: chunks.length, embeddings: embeddings.length, setId };
|
|
4426
|
+
const embeddingSets = new EmbeddingSetsRepository(db);
|
|
4427
|
+
const set = await embeddingSets.ensureDefault();
|
|
4428
|
+
await embeddingSets.putEmbedding({
|
|
4429
|
+
note_id: job.note_id,
|
|
4430
|
+
embedding_set_id: set.id,
|
|
4431
|
+
vector: vector2
|
|
4432
|
+
});
|
|
4433
|
+
return { chunks: chunks.length, embeddings: embeddings.length, setId: set.id };
|
|
3539
4434
|
}
|
|
3540
4435
|
|
|
3541
4436
|
// src/capabilities/auto-tag.ts
|
|
@@ -4424,16 +5319,16 @@ function createCspReportHandler(onReport) {
|
|
|
4424
5319
|
if (request.method !== "POST") {
|
|
4425
5320
|
return new Response(null, { status: 405, headers: { Allow: "POST" } });
|
|
4426
5321
|
}
|
|
4427
|
-
let
|
|
5322
|
+
let json2;
|
|
4428
5323
|
try {
|
|
4429
|
-
|
|
5324
|
+
json2 = await request.json();
|
|
4430
5325
|
} catch {
|
|
4431
5326
|
return new Response(JSON.stringify({ error: "Invalid CSP report JSON" }), {
|
|
4432
5327
|
status: 400,
|
|
4433
5328
|
headers: { "Content-Type": "application/json" }
|
|
4434
5329
|
});
|
|
4435
5330
|
}
|
|
4436
|
-
await onReport(parseCspReport(
|
|
5331
|
+
await onReport(parseCspReport(json2));
|
|
4437
5332
|
return new Response(null, { status: 204 });
|
|
4438
5333
|
};
|
|
4439
5334
|
}
|
|
@@ -4647,17 +5542,39 @@ function tagsFromShard(shardTags) {
|
|
|
4647
5542
|
function embeddingSetToShard(set) {
|
|
4648
5543
|
return {
|
|
4649
5544
|
id: set.id,
|
|
5545
|
+
name: set.name ?? set.model_name,
|
|
5546
|
+
purpose: set.purpose ?? null,
|
|
4650
5547
|
model: set.model_name,
|
|
4651
5548
|
dimension: set.dimensions,
|
|
4652
|
-
|
|
5549
|
+
kind: set.kind ?? "physical",
|
|
5550
|
+
mode: set.mode ?? null,
|
|
5551
|
+
truncate_dimension: set.truncate_dimension ?? null,
|
|
5552
|
+
criteria: jsonObject2(set.criteria_json),
|
|
5553
|
+
source: jsonObject2(set.source_json),
|
|
5554
|
+
compatibility: jsonObject2(set.compatibility_json),
|
|
5555
|
+
materialization: jsonObject2(set.materialization_json),
|
|
5556
|
+
freshness: jsonObject2(set.freshness_json),
|
|
5557
|
+
created_at: toISOString(set.created_at),
|
|
5558
|
+
updated_at: set.updated_at ? toISOString(set.updated_at) : void 0
|
|
4653
5559
|
};
|
|
4654
5560
|
}
|
|
4655
5561
|
function embeddingSetFromShard(shard) {
|
|
4656
5562
|
return {
|
|
4657
5563
|
id: shard.id,
|
|
5564
|
+
name: shard.name ?? shard.model,
|
|
5565
|
+
purpose: shard.purpose ?? null,
|
|
4658
5566
|
model_name: shard.model,
|
|
4659
5567
|
dimensions: shard.dimension,
|
|
4660
|
-
|
|
5568
|
+
kind: shard.kind ?? "physical",
|
|
5569
|
+
mode: shard.mode ?? null,
|
|
5570
|
+
truncate_dimension: shard.truncate_dimension ?? null,
|
|
5571
|
+
criteria_json: jsonString(shard.criteria),
|
|
5572
|
+
source_json: jsonString(shard.source),
|
|
5573
|
+
compatibility_json: jsonString(shard.compatibility),
|
|
5574
|
+
materialization_json: jsonString(shard.materialization),
|
|
5575
|
+
freshness_json: jsonString(shard.freshness),
|
|
5576
|
+
created_at: shard.created_at,
|
|
5577
|
+
updated_at: shard.updated_at ?? null
|
|
4661
5578
|
};
|
|
4662
5579
|
}
|
|
4663
5580
|
function embeddingSetMemberToShard(member) {
|
|
@@ -4685,6 +5602,55 @@ function embeddingFromShard(shard) {
|
|
|
4685
5602
|
created_at: shard.created_at
|
|
4686
5603
|
};
|
|
4687
5604
|
}
|
|
5605
|
+
function skosSchemeToShard(scheme) {
|
|
5606
|
+
return {
|
|
5607
|
+
id: scheme.id,
|
|
5608
|
+
title: scheme.title,
|
|
5609
|
+
description: scheme.description,
|
|
5610
|
+
created_at: toISOString(scheme.created_at),
|
|
5611
|
+
updated_at: toISOString(scheme.updated_at)
|
|
5612
|
+
};
|
|
5613
|
+
}
|
|
5614
|
+
function skosConceptToShard(concept) {
|
|
5615
|
+
return {
|
|
5616
|
+
id: concept.id,
|
|
5617
|
+
scheme_id: concept.scheme_id,
|
|
5618
|
+
pref_label: concept.pref_label,
|
|
5619
|
+
alt_labels: parseJsonArrayField(concept.alt_labels),
|
|
5620
|
+
definition: concept.definition,
|
|
5621
|
+
created_at: toISOString(concept.created_at),
|
|
5622
|
+
updated_at: toISOString(concept.updated_at)
|
|
5623
|
+
};
|
|
5624
|
+
}
|
|
5625
|
+
function skosRelationToShard(relation) {
|
|
5626
|
+
return {
|
|
5627
|
+
id: relation.id,
|
|
5628
|
+
source_concept_id: relation.source_concept_id,
|
|
5629
|
+
target_concept_id: relation.target_concept_id,
|
|
5630
|
+
relation_type: relation.relation_type,
|
|
5631
|
+
created_at: toISOString(relation.created_at)
|
|
5632
|
+
};
|
|
5633
|
+
}
|
|
5634
|
+
function noteSkosTagToShard(tag) {
|
|
5635
|
+
return {
|
|
5636
|
+
id: tag.id,
|
|
5637
|
+
note_id: tag.note_id,
|
|
5638
|
+
concept_id: tag.concept_id,
|
|
5639
|
+
created_at: toISOString(tag.created_at)
|
|
5640
|
+
};
|
|
5641
|
+
}
|
|
5642
|
+
function provenanceEdgeToShard(edge) {
|
|
5643
|
+
return {
|
|
5644
|
+
id: edge.id,
|
|
5645
|
+
entity_type: edge.entity_type,
|
|
5646
|
+
entity_id: edge.entity_id,
|
|
5647
|
+
activity: edge.activity,
|
|
5648
|
+
agent: edge.agent,
|
|
5649
|
+
started_at: toISOString(edge.started_at),
|
|
5650
|
+
ended_at: edge.ended_at ? toISOString(edge.ended_at) : null,
|
|
5651
|
+
attributes: parseJsonObjectField(edge.attributes)
|
|
5652
|
+
};
|
|
5653
|
+
}
|
|
4688
5654
|
function toISOString(date) {
|
|
4689
5655
|
if (date instanceof Date) return date.toISOString();
|
|
4690
5656
|
return date;
|
|
@@ -4693,9 +5659,37 @@ function parseVector(vectorStr) {
|
|
|
4693
5659
|
const inner = vectorStr.replace(/^\[/, "").replace(/\]$/, "");
|
|
4694
5660
|
return inner.split(",").map(Number);
|
|
4695
5661
|
}
|
|
5662
|
+
function parseJsonArrayField(value) {
|
|
5663
|
+
if (Array.isArray(value)) return value;
|
|
5664
|
+
if (!value) return [];
|
|
5665
|
+
const parsed = JSON.parse(value);
|
|
5666
|
+
return Array.isArray(parsed) ? parsed.map(String) : [];
|
|
5667
|
+
}
|
|
5668
|
+
function parseJsonObjectField(value) {
|
|
5669
|
+
if (!value) return null;
|
|
5670
|
+
if (typeof value !== "string") return value;
|
|
5671
|
+
const parsed = JSON.parse(value);
|
|
5672
|
+
return parsed && typeof parsed === "object" && !Array.isArray(parsed) ? parsed : null;
|
|
5673
|
+
}
|
|
5674
|
+
function jsonObject2(value) {
|
|
5675
|
+
if (value == null) return null;
|
|
5676
|
+
if (typeof value === "string") return JSON.parse(value);
|
|
5677
|
+
return value;
|
|
5678
|
+
}
|
|
5679
|
+
function jsonString(value) {
|
|
5680
|
+
return value == null ? null : JSON.stringify(value);
|
|
5681
|
+
}
|
|
4696
5682
|
|
|
4697
5683
|
// src/shard/shard-export.ts
|
|
4698
5684
|
var encoder = new TextEncoder();
|
|
5685
|
+
function jsonObject3(value) {
|
|
5686
|
+
if (value == null) return void 0;
|
|
5687
|
+
if (typeof value === "string") return JSON.parse(value);
|
|
5688
|
+
return value;
|
|
5689
|
+
}
|
|
5690
|
+
function iso2(value) {
|
|
5691
|
+
return value instanceof Date ? value.toISOString() : value;
|
|
5692
|
+
}
|
|
4699
5693
|
async function exportShard(db, options) {
|
|
4700
5694
|
const files = /* @__PURE__ */ new Map();
|
|
4701
5695
|
const components = [];
|
|
@@ -4797,6 +5791,41 @@ async function exportShard(db, options) {
|
|
|
4797
5791
|
files.set("links.jsonl", encoder.encode(linksJsonl));
|
|
4798
5792
|
components.push("links");
|
|
4799
5793
|
counts.links = filteredLinks.length;
|
|
5794
|
+
const allNoteSkosRows = await db.query(`SELECT * FROM note_skos_tag ORDER BY created_at`);
|
|
5795
|
+
const filteredNoteSkosRows = isFiltered ? allNoteSkosRows.rows.filter((row) => exportedNoteIds.has(row.note_id)) : allNoteSkosRows.rows;
|
|
5796
|
+
const referencedConceptIds = new Set(filteredNoteSkosRows.map((row) => row.concept_id));
|
|
5797
|
+
const allConceptRows = await db.query(`SELECT * FROM skos_concept WHERE deleted_at IS NULL ORDER BY pref_label`);
|
|
5798
|
+
const filteredConceptRows = isFiltered ? allConceptRows.rows.filter((row) => referencedConceptIds.has(row.id)) : allConceptRows.rows;
|
|
5799
|
+
const exportedConceptIds = new Set(filteredConceptRows.map((row) => row.id));
|
|
5800
|
+
const exportedSchemeIds = new Set(filteredConceptRows.map((row) => row.scheme_id));
|
|
5801
|
+
const allSchemeRows = await db.query(`SELECT * FROM skos_scheme WHERE deleted_at IS NULL ORDER BY title`);
|
|
5802
|
+
const filteredSchemeRows = isFiltered ? allSchemeRows.rows.filter((row) => exportedSchemeIds.has(row.id)) : allSchemeRows.rows;
|
|
5803
|
+
const allRelationRows = await db.query(`SELECT * FROM skos_concept_relation ORDER BY created_at`);
|
|
5804
|
+
const filteredRelationRows = isFiltered ? allRelationRows.rows.filter(
|
|
5805
|
+
(row) => exportedConceptIds.has(row.source_concept_id) && exportedConceptIds.has(row.target_concept_id)
|
|
5806
|
+
) : allRelationRows.rows;
|
|
5807
|
+
const shardSkosSchemes = filteredSchemeRows.map(skosSchemeToShard);
|
|
5808
|
+
files.set("skos_schemes.json", encoder.encode(JSON.stringify(shardSkosSchemes)));
|
|
5809
|
+
components.push("skos_schemes");
|
|
5810
|
+
counts.skos_schemes = shardSkosSchemes.length;
|
|
5811
|
+
const shardSkosConcepts = filteredConceptRows.map(skosConceptToShard);
|
|
5812
|
+
files.set("skos_concepts.json", encoder.encode(JSON.stringify(shardSkosConcepts)));
|
|
5813
|
+
components.push("skos_concepts");
|
|
5814
|
+
counts.skos_concepts = shardSkosConcepts.length;
|
|
5815
|
+
const skosRelationsJsonl = filteredRelationRows.map((row) => JSON.stringify(skosRelationToShard(row))).join("\n");
|
|
5816
|
+
files.set("skos_relations.jsonl", encoder.encode(skosRelationsJsonl));
|
|
5817
|
+
components.push("skos_relations");
|
|
5818
|
+
counts.skos_relations = filteredRelationRows.length;
|
|
5819
|
+
const noteSkosJsonl = filteredNoteSkosRows.map((row) => JSON.stringify(noteSkosTagToShard(row))).join("\n");
|
|
5820
|
+
files.set("note_skos_tags.jsonl", encoder.encode(noteSkosJsonl));
|
|
5821
|
+
components.push("note_skos_tags");
|
|
5822
|
+
counts.note_skos_tags = filteredNoteSkosRows.length;
|
|
5823
|
+
const provenanceRows = await db.query(`SELECT * FROM provenance_edge ORDER BY started_at`);
|
|
5824
|
+
const filteredProvenanceRows = isFiltered ? provenanceRows.rows.filter((row) => row.entity_type !== "note" || exportedNoteIds.has(row.entity_id)) : provenanceRows.rows;
|
|
5825
|
+
const provenanceJsonl = filteredProvenanceRows.map((row) => JSON.stringify(provenanceEdgeToShard(row))).join("\n");
|
|
5826
|
+
files.set("provenance_edges.jsonl", encoder.encode(provenanceJsonl));
|
|
5827
|
+
components.push("provenance_edges");
|
|
5828
|
+
counts.provenance_edges = filteredProvenanceRows.length;
|
|
4800
5829
|
if (options?.includeEmbeddings) {
|
|
4801
5830
|
const embSetRows = await db.query(`SELECT * FROM embedding_set ORDER BY created_at`);
|
|
4802
5831
|
const shardEmbSets = embSetRows.rows.map(embeddingSetToShard);
|
|
@@ -4814,6 +5843,92 @@ async function exportShard(db, options) {
|
|
|
4814
5843
|
components.push("embeddings");
|
|
4815
5844
|
counts.embeddings = embRows.rows.length;
|
|
4816
5845
|
}
|
|
5846
|
+
const graphSourceRows = await db.query(`SELECT * FROM graph_source ORDER BY created_at, id`);
|
|
5847
|
+
if (graphSourceRows.rows.length > 0) {
|
|
5848
|
+
const shardGraphSources = graphSourceRows.rows.map((row) => ({
|
|
5849
|
+
id: row.id,
|
|
5850
|
+
name: row.name,
|
|
5851
|
+
kind: row.kind,
|
|
5852
|
+
source_table: row.source_table,
|
|
5853
|
+
embedding_set_id: row.embedding_set_id,
|
|
5854
|
+
virtual_set_id: row.virtual_set_id,
|
|
5855
|
+
model: row.model,
|
|
5856
|
+
dimension: row.dimension,
|
|
5857
|
+
truncate_dimension: row.truncate_dimension,
|
|
5858
|
+
metric: row.metric,
|
|
5859
|
+
algorithm: row.algorithm,
|
|
5860
|
+
parameters: jsonObject3(row.parameters_json),
|
|
5861
|
+
input_hash: row.input_hash,
|
|
5862
|
+
freshness: jsonObject3(row.freshness_json) ?? { status: "unknown" },
|
|
5863
|
+
created_at: iso2(row.created_at)
|
|
5864
|
+
}));
|
|
5865
|
+
files.set("graph_sources.json", encoder.encode(JSON.stringify(shardGraphSources)));
|
|
5866
|
+
components.push("graph_sources");
|
|
5867
|
+
counts.graph_sources = shardGraphSources.length;
|
|
5868
|
+
}
|
|
5869
|
+
const graphEdgeRows = await db.query(`SELECT * FROM graph_edge_artifact ORDER BY graph_source_id, from_note_id, to_note_id, kind`);
|
|
5870
|
+
if (graphEdgeRows.rows.length > 0) {
|
|
5871
|
+
const graphEdgesJsonl = graphEdgeRows.rows.map((row) => JSON.stringify({
|
|
5872
|
+
graph_source_id: row.graph_source_id,
|
|
5873
|
+
from_note_id: row.from_note_id,
|
|
5874
|
+
to_note_id: row.to_note_id,
|
|
5875
|
+
weight: row.weight,
|
|
5876
|
+
kind: row.kind,
|
|
5877
|
+
rank: row.rank,
|
|
5878
|
+
metadata: jsonObject3(row.metadata_json)
|
|
5879
|
+
})).join("\n");
|
|
5880
|
+
files.set("graph_edges.jsonl", encoder.encode(graphEdgesJsonl));
|
|
5881
|
+
components.push("graph_edges");
|
|
5882
|
+
counts.graph_edges = graphEdgeRows.rows.length;
|
|
5883
|
+
}
|
|
5884
|
+
const communitySetRows = await db.query(`SELECT * FROM community_set ORDER BY created_at, id`);
|
|
5885
|
+
const communityRows = await db.query(`SELECT * FROM community ORDER BY community_set_id, rank NULLS LAST, id`);
|
|
5886
|
+
if (communitySetRows.rows.length > 0) {
|
|
5887
|
+
const communitiesBySet = /* @__PURE__ */ new Map();
|
|
5888
|
+
for (const row of communityRows.rows) {
|
|
5889
|
+
const rows = communitiesBySet.get(row.community_set_id) ?? [];
|
|
5890
|
+
rows.push(row);
|
|
5891
|
+
communitiesBySet.set(row.community_set_id, rows);
|
|
5892
|
+
}
|
|
5893
|
+
const shardCommunitySets = communitySetRows.rows.map((row) => ({
|
|
5894
|
+
id: row.id,
|
|
5895
|
+
graph_source_id: row.graph_source_id,
|
|
5896
|
+
name: row.name,
|
|
5897
|
+
source_type: row.source_type,
|
|
5898
|
+
algorithm: row.algorithm,
|
|
5899
|
+
parameters: jsonObject3(row.parameters_json),
|
|
5900
|
+
input_hash: row.input_hash,
|
|
5901
|
+
freshness: jsonObject3(row.freshness_json) ?? { status: "unknown" },
|
|
5902
|
+
communities: (communitiesBySet.get(row.id) ?? []).map((community) => ({
|
|
5903
|
+
id: community.id,
|
|
5904
|
+
label: community.label,
|
|
5905
|
+
rank: community.rank,
|
|
5906
|
+
size: community.size,
|
|
5907
|
+
confidence: community.confidence,
|
|
5908
|
+
representative_note_ids: community.representative_note_ids ?? [],
|
|
5909
|
+
metadata: jsonObject3(community.metadata_json)
|
|
5910
|
+
})),
|
|
5911
|
+
created_at: iso2(row.created_at)
|
|
5912
|
+
}));
|
|
5913
|
+
files.set("communities.json", encoder.encode(JSON.stringify(shardCommunitySets)));
|
|
5914
|
+
components.push("communities");
|
|
5915
|
+
counts.community_sets = shardCommunitySets.length;
|
|
5916
|
+
counts.communities = communityRows.rows.length;
|
|
5917
|
+
}
|
|
5918
|
+
const assignmentRows = await db.query(`SELECT * FROM community_assignment ORDER BY community_set_id, community_id, note_id`);
|
|
5919
|
+
if (assignmentRows.rows.length > 0) {
|
|
5920
|
+
const assignmentsJsonl = assignmentRows.rows.map((row) => JSON.stringify({
|
|
5921
|
+
community_set_id: row.community_set_id,
|
|
5922
|
+
community_id: row.community_id,
|
|
5923
|
+
note_id: row.note_id,
|
|
5924
|
+
confidence: row.confidence,
|
|
5925
|
+
source_type: row.source_type,
|
|
5926
|
+
metadata: jsonObject3(row.metadata_json)
|
|
5927
|
+
})).join("\n");
|
|
5928
|
+
files.set("community_assignments.jsonl", encoder.encode(assignmentsJsonl));
|
|
5929
|
+
components.push("community_assignments");
|
|
5930
|
+
counts.community_assignments = assignmentRows.rows.length;
|
|
5931
|
+
}
|
|
4817
5932
|
const checksums = {};
|
|
4818
5933
|
for (const [filename, data] of files) {
|
|
4819
5934
|
checksums[filename] = await sha256Hex(data);
|
|
@@ -4846,7 +5961,17 @@ async function importShard(db, data, options) {
|
|
|
4846
5961
|
links: 0,
|
|
4847
5962
|
embedding_sets: 0,
|
|
4848
5963
|
embedding_set_members: 0,
|
|
4849
|
-
embeddings: 0
|
|
5964
|
+
embeddings: 0,
|
|
5965
|
+
skos_schemes: 0,
|
|
5966
|
+
skos_concepts: 0,
|
|
5967
|
+
skos_relations: 0,
|
|
5968
|
+
note_skos_tags: 0,
|
|
5969
|
+
provenance_edges: 0,
|
|
5970
|
+
graph_sources: 0,
|
|
5971
|
+
graph_edges: 0,
|
|
5972
|
+
community_sets: 0,
|
|
5973
|
+
communities: 0,
|
|
5974
|
+
community_assignments: 0
|
|
4850
5975
|
};
|
|
4851
5976
|
const skipped = {};
|
|
4852
5977
|
const inputData = data instanceof ArrayBuffer ? new Uint8Array(data) : data;
|
|
@@ -4919,6 +6044,15 @@ async function importShard(db, data, options) {
|
|
|
4919
6044
|
files.get("embedding_set_members.jsonl")
|
|
4920
6045
|
);
|
|
4921
6046
|
const parsedEmbeddings = parseJsonl(files.get("embeddings.jsonl"));
|
|
6047
|
+
const parsedSkosSchemes = parseJsonArray(files.get("skos_schemes.json"));
|
|
6048
|
+
const parsedSkosConcepts = parseJsonArray(files.get("skos_concepts.json"));
|
|
6049
|
+
const parsedSkosRelations = parseJsonl(files.get("skos_relations.jsonl"));
|
|
6050
|
+
const parsedNoteSkosTags = parseJsonl(files.get("note_skos_tags.jsonl"));
|
|
6051
|
+
const parsedProvenanceEdges = parseJsonl(files.get("provenance_edges.jsonl"));
|
|
6052
|
+
const parsedGraphSources = parseJsonArray(files.get("graph_sources.json"));
|
|
6053
|
+
const parsedGraphEdges = parseJsonl(files.get("graph_edges.jsonl"));
|
|
6054
|
+
const parsedCommunitySets = parseJsonArray(files.get("communities.json"));
|
|
6055
|
+
const parsedCommunityAssignments = parseJsonl(files.get("community_assignments.jsonl"));
|
|
4922
6056
|
const knownFiles = /* @__PURE__ */ new Set([
|
|
4923
6057
|
"manifest.json",
|
|
4924
6058
|
"notes.jsonl",
|
|
@@ -4929,7 +6063,16 @@ async function importShard(db, data, options) {
|
|
|
4929
6063
|
"embedding_set_members.jsonl",
|
|
4930
6064
|
"embedding_configs.json",
|
|
4931
6065
|
"embeddings.jsonl",
|
|
4932
|
-
"templates.json"
|
|
6066
|
+
"templates.json",
|
|
6067
|
+
"skos_schemes.json",
|
|
6068
|
+
"skos_concepts.json",
|
|
6069
|
+
"skos_relations.jsonl",
|
|
6070
|
+
"note_skos_tags.jsonl",
|
|
6071
|
+
"provenance_edges.jsonl",
|
|
6072
|
+
"graph_sources.json",
|
|
6073
|
+
"graph_edges.jsonl",
|
|
6074
|
+
"communities.json",
|
|
6075
|
+
"community_assignments.jsonl"
|
|
4933
6076
|
]);
|
|
4934
6077
|
for (const filename of files.keys()) {
|
|
4935
6078
|
if (!knownFiles.has(filename)) {
|
|
@@ -5038,6 +6181,41 @@ async function importShard(db, data, options) {
|
|
|
5038
6181
|
}
|
|
5039
6182
|
counts.notes++;
|
|
5040
6183
|
}
|
|
6184
|
+
for (const scheme of parsedSkosSchemes) {
|
|
6185
|
+
if (strategy === "replace") {
|
|
6186
|
+
await tx.query(
|
|
6187
|
+
`INSERT INTO skos_scheme (id, title, description, created_at, updated_at)
|
|
6188
|
+
VALUES ($1, $2, $3, $4, $5)
|
|
6189
|
+
ON CONFLICT (id) DO UPDATE SET title = $2, description = $3, updated_at = $5`,
|
|
6190
|
+
[scheme.id, scheme.title, scheme.description, scheme.created_at, scheme.updated_at]
|
|
6191
|
+
);
|
|
6192
|
+
} else {
|
|
6193
|
+
await tx.query(
|
|
6194
|
+
`INSERT INTO skos_scheme (id, title, description, created_at, updated_at)
|
|
6195
|
+
VALUES ($1, $2, $3, $4, $5) ${conflictClause}`,
|
|
6196
|
+
[scheme.id, scheme.title, scheme.description, scheme.created_at, scheme.updated_at]
|
|
6197
|
+
);
|
|
6198
|
+
}
|
|
6199
|
+
counts.skos_schemes++;
|
|
6200
|
+
}
|
|
6201
|
+
for (const concept of parsedSkosConcepts) {
|
|
6202
|
+
const altLabels = JSON.stringify(concept.alt_labels ?? []);
|
|
6203
|
+
if (strategy === "replace") {
|
|
6204
|
+
await tx.query(
|
|
6205
|
+
`INSERT INTO skos_concept (id, scheme_id, pref_label, alt_labels, definition, created_at, updated_at)
|
|
6206
|
+
VALUES ($1, $2, $3, $4, $5, $6, $7)
|
|
6207
|
+
ON CONFLICT (id) DO UPDATE SET scheme_id = $2, pref_label = $3, alt_labels = $4, definition = $5, updated_at = $7`,
|
|
6208
|
+
[concept.id, concept.scheme_id, concept.pref_label, altLabels, concept.definition, concept.created_at, concept.updated_at]
|
|
6209
|
+
);
|
|
6210
|
+
} else {
|
|
6211
|
+
await tx.query(
|
|
6212
|
+
`INSERT INTO skos_concept (id, scheme_id, pref_label, alt_labels, definition, created_at, updated_at)
|
|
6213
|
+
VALUES ($1, $2, $3, $4, $5, $6, $7) ${conflictClause}`,
|
|
6214
|
+
[concept.id, concept.scheme_id, concept.pref_label, altLabels, concept.definition, concept.created_at, concept.updated_at]
|
|
6215
|
+
);
|
|
6216
|
+
}
|
|
6217
|
+
counts.skos_concepts++;
|
|
6218
|
+
}
|
|
5041
6219
|
for (const shardLink of parsedLinks) {
|
|
5042
6220
|
const link = linkFromShard(shardLink);
|
|
5043
6221
|
if (strategy === "replace") {
|
|
@@ -5056,20 +6234,70 @@ async function importShard(db, data, options) {
|
|
|
5056
6234
|
}
|
|
5057
6235
|
counts.links++;
|
|
5058
6236
|
}
|
|
6237
|
+
for (const relation of parsedSkosRelations) {
|
|
6238
|
+
if (strategy === "replace") {
|
|
6239
|
+
await tx.query(
|
|
6240
|
+
`INSERT INTO skos_concept_relation (id, source_concept_id, target_concept_id, relation_type, created_at)
|
|
6241
|
+
VALUES ($1, $2, $3, $4, $5)
|
|
6242
|
+
ON CONFLICT (id) DO UPDATE SET source_concept_id = $2, target_concept_id = $3, relation_type = $4`,
|
|
6243
|
+
[relation.id, relation.source_concept_id, relation.target_concept_id, relation.relation_type, relation.created_at]
|
|
6244
|
+
);
|
|
6245
|
+
} else {
|
|
6246
|
+
await tx.query(
|
|
6247
|
+
`INSERT INTO skos_concept_relation (id, source_concept_id, target_concept_id, relation_type, created_at)
|
|
6248
|
+
VALUES ($1, $2, $3, $4, $5) ${conflictClause}`,
|
|
6249
|
+
[relation.id, relation.source_concept_id, relation.target_concept_id, relation.relation_type, relation.created_at]
|
|
6250
|
+
);
|
|
6251
|
+
}
|
|
6252
|
+
counts.skos_relations++;
|
|
6253
|
+
}
|
|
6254
|
+
for (const tag of parsedNoteSkosTags) {
|
|
6255
|
+
await tx.query(
|
|
6256
|
+
`INSERT INTO note_skos_tag (id, note_id, concept_id, created_at)
|
|
6257
|
+
VALUES ($1, $2, $3, $4)
|
|
6258
|
+
ON CONFLICT (note_id, concept_id) DO NOTHING`,
|
|
6259
|
+
[tag.id, tag.note_id, tag.concept_id, tag.created_at]
|
|
6260
|
+
);
|
|
6261
|
+
counts.note_skos_tags++;
|
|
6262
|
+
}
|
|
6263
|
+
for (const edge of parsedProvenanceEdges) {
|
|
6264
|
+
const attributes = edge.attributes === null ? null : JSON.stringify(edge.attributes);
|
|
6265
|
+
if (strategy === "replace") {
|
|
6266
|
+
await tx.query(
|
|
6267
|
+
`INSERT INTO provenance_edge (id, entity_type, entity_id, activity, agent, started_at, ended_at, attributes)
|
|
6268
|
+
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
|
|
6269
|
+
ON CONFLICT (id) DO UPDATE SET entity_type = $2, entity_id = $3, activity = $4, agent = $5, started_at = $6, ended_at = $7, attributes = $8`,
|
|
6270
|
+
[edge.id, edge.entity_type, edge.entity_id, edge.activity, edge.agent, edge.started_at, edge.ended_at, attributes]
|
|
6271
|
+
);
|
|
6272
|
+
} else {
|
|
6273
|
+
await tx.query(
|
|
6274
|
+
`INSERT INTO provenance_edge (id, entity_type, entity_id, activity, agent, started_at, ended_at, attributes)
|
|
6275
|
+
VALUES ($1, $2, $3, $4, $5, $6, $7, $8) ${conflictClause}`,
|
|
6276
|
+
[edge.id, edge.entity_type, edge.entity_id, edge.activity, edge.agent, edge.started_at, edge.ended_at, attributes]
|
|
6277
|
+
);
|
|
6278
|
+
}
|
|
6279
|
+
counts.provenance_edges++;
|
|
6280
|
+
}
|
|
5059
6281
|
for (const shardSet of parsedEmbSets) {
|
|
5060
6282
|
const set = embeddingSetFromShard(shardSet);
|
|
5061
6283
|
if (strategy === "replace") {
|
|
5062
6284
|
await tx.query(
|
|
5063
|
-
`INSERT INTO embedding_set (
|
|
5064
|
-
|
|
5065
|
-
|
|
5066
|
-
|
|
6285
|
+
`INSERT INTO embedding_set (
|
|
6286
|
+
id, name, purpose, model_name, dimensions, kind, mode, truncate_dimension,
|
|
6287
|
+
criteria_json, source_json, compatibility_json, materialization_json, freshness_json, created_at, updated_at
|
|
6288
|
+
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::jsonb, $10::jsonb, $11::jsonb, $12::jsonb, $13::jsonb, $14, COALESCE($15::timestamptz, $14::timestamptz))
|
|
6289
|
+
ON CONFLICT (id) DO UPDATE SET name = $2, purpose = $3, model_name = $4, dimensions = $5,
|
|
6290
|
+
kind = $6, mode = $7, truncate_dimension = $8, criteria_json = $9::jsonb, source_json = $10::jsonb,
|
|
6291
|
+
compatibility_json = $11::jsonb, materialization_json = $12::jsonb, freshness_json = $13::jsonb, updated_at = COALESCE($15::timestamptz, $14::timestamptz)`,
|
|
6292
|
+
[set.id, set.name, set.purpose, set.model_name, set.dimensions, set.kind, set.mode, set.truncate_dimension, set.criteria_json, set.source_json, set.compatibility_json, set.materialization_json, set.freshness_json, set.created_at, set.updated_at]
|
|
5067
6293
|
);
|
|
5068
6294
|
} else {
|
|
5069
6295
|
await tx.query(
|
|
5070
|
-
`INSERT INTO embedding_set (
|
|
5071
|
-
|
|
5072
|
-
|
|
6296
|
+
`INSERT INTO embedding_set (
|
|
6297
|
+
id, name, purpose, model_name, dimensions, kind, mode, truncate_dimension,
|
|
6298
|
+
criteria_json, source_json, compatibility_json, materialization_json, freshness_json, created_at, updated_at
|
|
6299
|
+
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::jsonb, $10::jsonb, $11::jsonb, $12::jsonb, $13::jsonb, $14, COALESCE($15::timestamptz, $14::timestamptz)) ${conflictClause}`,
|
|
6300
|
+
[set.id, set.name, set.purpose, set.model_name, set.dimensions, set.kind, set.mode, set.truncate_dimension, set.criteria_json, set.source_json, set.compatibility_json, set.materialization_json, set.freshness_json, set.created_at, set.updated_at]
|
|
5073
6301
|
);
|
|
5074
6302
|
}
|
|
5075
6303
|
counts.embedding_sets++;
|
|
@@ -5092,6 +6320,60 @@ async function importShard(db, data, options) {
|
|
|
5092
6320
|
}
|
|
5093
6321
|
counts.embeddings++;
|
|
5094
6322
|
}
|
|
6323
|
+
for (const source of parsedGraphSources) {
|
|
6324
|
+
const parameters = source.parameters == null ? null : JSON.stringify(source.parameters);
|
|
6325
|
+
const freshness = JSON.stringify({ ...source.freshness ?? {}, status: "unknown" });
|
|
6326
|
+
await tx.query(
|
|
6327
|
+
`INSERT INTO graph_source (
|
|
6328
|
+
id, name, kind, source_table, embedding_set_id, virtual_set_id, model, dimension,
|
|
6329
|
+
truncate_dimension, metric, algorithm, parameters_json, input_hash, freshness_json, created_at
|
|
6330
|
+
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12::jsonb, $13, $14::jsonb, $15)
|
|
6331
|
+
${strategy === "replace" ? "ON CONFLICT (id) DO UPDATE SET name = $2, kind = $3, source_table = $4, embedding_set_id = $5, virtual_set_id = $6, model = $7, dimension = $8, truncate_dimension = $9, metric = $10, algorithm = $11, parameters_json = $12::jsonb, input_hash = $13, freshness_json = $14::jsonb, created_at = $15" : conflictClause}`,
|
|
6332
|
+
[source.id, source.name, source.kind, source.source_table ?? null, source.embedding_set_id ?? null, source.virtual_set_id ?? null, source.model ?? null, source.dimension ?? null, source.truncate_dimension ?? null, source.metric ?? null, source.algorithm ?? null, parameters, source.input_hash, freshness, source.created_at]
|
|
6333
|
+
);
|
|
6334
|
+
counts.graph_sources++;
|
|
6335
|
+
}
|
|
6336
|
+
for (const edge of parsedGraphEdges) {
|
|
6337
|
+
const metadata = edge.metadata == null ? null : JSON.stringify(edge.metadata);
|
|
6338
|
+
await tx.query(
|
|
6339
|
+
`INSERT INTO graph_edge_artifact (graph_source_id, from_note_id, to_note_id, weight, kind, rank, metadata_json)
|
|
6340
|
+
VALUES ($1, $2, $3, $4, $5, $6, $7::jsonb)
|
|
6341
|
+
${strategy === "replace" ? "ON CONFLICT (graph_source_id, from_note_id, to_note_id, kind) DO UPDATE SET weight = $4, rank = $6, metadata_json = $7::jsonb" : conflictClause}`,
|
|
6342
|
+
[edge.graph_source_id, edge.from_note_id, edge.to_note_id, edge.weight, edge.kind, edge.rank ?? null, metadata]
|
|
6343
|
+
);
|
|
6344
|
+
counts.graph_edges++;
|
|
6345
|
+
}
|
|
6346
|
+
for (const set of parsedCommunitySets) {
|
|
6347
|
+
const parameters = set.parameters == null ? null : JSON.stringify(set.parameters);
|
|
6348
|
+
const freshness = JSON.stringify({ ...set.freshness ?? {}, status: "unknown" });
|
|
6349
|
+
await tx.query(
|
|
6350
|
+
`INSERT INTO community_set (id, graph_source_id, name, source_type, algorithm, parameters_json, input_hash, freshness_json, created_at)
|
|
6351
|
+
VALUES ($1, $2, $3, $4, $5, $6::jsonb, $7, $8::jsonb, $9)
|
|
6352
|
+
${strategy === "replace" ? "ON CONFLICT (id) DO UPDATE SET graph_source_id = $2, name = $3, source_type = $4, algorithm = $5, parameters_json = $6::jsonb, input_hash = $7, freshness_json = $8::jsonb, created_at = $9" : conflictClause}`,
|
|
6353
|
+
[set.id, set.graph_source_id, set.name, set.source_type, set.algorithm ?? null, parameters, set.input_hash, freshness, set.created_at]
|
|
6354
|
+
);
|
|
6355
|
+
counts.community_sets++;
|
|
6356
|
+
for (const community of set.communities ?? []) {
|
|
6357
|
+
const metadata = community.metadata == null ? null : JSON.stringify(community.metadata);
|
|
6358
|
+
await tx.query(
|
|
6359
|
+
`INSERT INTO community (community_set_id, id, label, rank, size, confidence, representative_note_ids, metadata_json)
|
|
6360
|
+
VALUES ($1, $2, $3, $4, $5, $6, $7, $8::jsonb)
|
|
6361
|
+
${strategy === "replace" ? "ON CONFLICT (community_set_id, id) DO UPDATE SET label = $3, rank = $4, size = $5, confidence = $6, representative_note_ids = $7, metadata_json = $8::jsonb" : conflictClause}`,
|
|
6362
|
+
[set.id, community.id, community.label ?? null, community.rank ?? null, community.size ?? null, community.confidence ?? null, community.representative_note_ids ?? [], metadata]
|
|
6363
|
+
);
|
|
6364
|
+
counts.communities++;
|
|
6365
|
+
}
|
|
6366
|
+
}
|
|
6367
|
+
for (const assignment of parsedCommunityAssignments) {
|
|
6368
|
+
const metadata = assignment.metadata == null ? null : JSON.stringify(assignment.metadata);
|
|
6369
|
+
await tx.query(
|
|
6370
|
+
`INSERT INTO community_assignment (community_set_id, community_id, note_id, confidence, source_type, metadata_json)
|
|
6371
|
+
VALUES ($1, $2, $3, $4, $5, $6::jsonb)
|
|
6372
|
+
${strategy === "replace" ? "ON CONFLICT (community_set_id, note_id) DO UPDATE SET community_id = $2, confidence = $4, source_type = $5, metadata_json = $6::jsonb" : conflictClause}`,
|
|
6373
|
+
[assignment.community_set_id, assignment.community_id, assignment.note_id, assignment.confidence ?? null, assignment.source_type, metadata]
|
|
6374
|
+
);
|
|
6375
|
+
counts.community_assignments++;
|
|
6376
|
+
}
|
|
5095
6377
|
for (const member of parsedEmbMembers) {
|
|
5096
6378
|
await tx.query(
|
|
5097
6379
|
`INSERT INTO embedding_set_member (embedding_set_id, note_id, embedding_id)
|
|
@@ -5130,9 +6412,144 @@ function parseJsonArray(data) {
|
|
|
5130
6412
|
return JSON.parse(decoder.decode(data));
|
|
5131
6413
|
}
|
|
5132
6414
|
|
|
6415
|
+
// src/aiwg-index.ts
|
|
6416
|
+
var REQUIRED_RECORD_FIELDS = [
|
|
6417
|
+
"schema_version",
|
|
6418
|
+
"id",
|
|
6419
|
+
"type",
|
|
6420
|
+
"source",
|
|
6421
|
+
"title",
|
|
6422
|
+
"text",
|
|
6423
|
+
"facets",
|
|
6424
|
+
"tags",
|
|
6425
|
+
"concepts",
|
|
6426
|
+
"relationships",
|
|
6427
|
+
"provenance",
|
|
6428
|
+
"privacy",
|
|
6429
|
+
"updated_at"
|
|
6430
|
+
];
|
|
6431
|
+
var VALID_TYPES = /* @__PURE__ */ new Set([
|
|
6432
|
+
"crm.contact",
|
|
6433
|
+
"crm.organization",
|
|
6434
|
+
"crm.event",
|
|
6435
|
+
"crm.interaction",
|
|
6436
|
+
"aiwg.artifact"
|
|
6437
|
+
]);
|
|
6438
|
+
function hasString(value) {
|
|
6439
|
+
return typeof value === "string" && value.length > 0;
|
|
6440
|
+
}
|
|
6441
|
+
function pushFacet(counts, name, value) {
|
|
6442
|
+
counts[name] ??= {};
|
|
6443
|
+
counts[name][value] = (counts[name][value] ?? 0) + 1;
|
|
6444
|
+
}
|
|
6445
|
+
function validateAiwgFortemiIndexExport(value) {
|
|
6446
|
+
const errors = [];
|
|
6447
|
+
const counts = {};
|
|
6448
|
+
const data = value;
|
|
6449
|
+
if (data?.schema_version !== "aiwg.fortemi.index.export.v1") {
|
|
6450
|
+
errors.push("schema_version must be aiwg.fortemi.index.export.v1");
|
|
6451
|
+
}
|
|
6452
|
+
if (!hasString(data?.generated_at)) errors.push("generated_at is required");
|
|
6453
|
+
if (!hasString(data?.source?.repo)) errors.push("source.repo is required");
|
|
6454
|
+
if (!hasString(data?.source?.privacy)) errors.push("source.privacy is required");
|
|
6455
|
+
if (!Array.isArray(data?.items)) errors.push("items must be an array");
|
|
6456
|
+
const ids = /* @__PURE__ */ new Set();
|
|
6457
|
+
let previousId = "";
|
|
6458
|
+
for (const [index, item] of (data.items ?? []).entries()) {
|
|
6459
|
+
for (const field of REQUIRED_RECORD_FIELDS) {
|
|
6460
|
+
if (!(field in item)) errors.push("items[" + index + "]." + field + " is required");
|
|
6461
|
+
}
|
|
6462
|
+
if (item.schema_version !== "aiwg.fortemi.index.record.v1") {
|
|
6463
|
+
errors.push("items[" + index + "].schema_version must be aiwg.fortemi.index.record.v1");
|
|
6464
|
+
}
|
|
6465
|
+
if (!hasString(item.id)) errors.push("items[" + index + "].id is required");
|
|
6466
|
+
if (hasString(item.id) && ids.has(item.id)) errors.push("duplicate id: " + item.id);
|
|
6467
|
+
if (hasString(item.id)) ids.add(item.id);
|
|
6468
|
+
if (previousId && hasString(item.id) && previousId.localeCompare(item.id) > 0) {
|
|
6469
|
+
errors.push("items must be sorted by id: " + previousId + " before " + item.id);
|
|
6470
|
+
}
|
|
6471
|
+
if (hasString(item.id)) previousId = item.id;
|
|
6472
|
+
if (!VALID_TYPES.has(item.type)) errors.push("items[" + index + "].type is invalid");
|
|
6473
|
+
else counts[item.type] = (counts[item.type] ?? 0) + 1;
|
|
6474
|
+
if (!hasString(item.source?.path)) errors.push("items[" + index + "].source.path is required");
|
|
6475
|
+
if (!hasString(item.source?.repo_relative_path)) errors.push("items[" + index + "].source.repo_relative_path is required");
|
|
6476
|
+
if (!hasString(item.source?.locator)) errors.push("items[" + index + "].source.locator is required");
|
|
6477
|
+
if (!Array.isArray(item.tags)) errors.push("items[" + index + "].tags must be an array");
|
|
6478
|
+
if (!Array.isArray(item.concepts)) errors.push("items[" + index + "].concepts must be an array");
|
|
6479
|
+
if (!Array.isArray(item.relationships)) errors.push("items[" + index + "].relationships must be an array");
|
|
6480
|
+
if (!Array.isArray(item.provenance) || item.provenance.length === 0) {
|
|
6481
|
+
errors.push("items[" + index + "].provenance must be a non-empty array");
|
|
6482
|
+
}
|
|
6483
|
+
if (!item.privacy || typeof item.privacy.pii !== "boolean" || !hasString(item.privacy.classification)) {
|
|
6484
|
+
errors.push("items[" + index + "].privacy requires classification and pii");
|
|
6485
|
+
}
|
|
6486
|
+
}
|
|
6487
|
+
return { valid: errors.length === 0, errors, counts };
|
|
6488
|
+
}
|
|
6489
|
+
function assertAiwgFortemiIndexExport(value) {
|
|
6490
|
+
const result = validateAiwgFortemiIndexExport(value);
|
|
6491
|
+
if (!result.valid) {
|
|
6492
|
+
throw new Error("Invalid AIWG Fortemi index export:\n" + result.errors.join("\n"));
|
|
6493
|
+
}
|
|
6494
|
+
return value;
|
|
6495
|
+
}
|
|
6496
|
+
function getAiwgFortemiFacets(items) {
|
|
6497
|
+
const result = {};
|
|
6498
|
+
for (const item of items) {
|
|
6499
|
+
pushFacet(result, "type", item.type);
|
|
6500
|
+
pushFacet(result, "privacy", item.privacy.classification);
|
|
6501
|
+
for (const tag of item.tags) pushFacet(result, "tag", tag);
|
|
6502
|
+
for (const concept of item.concepts) pushFacet(result, "concept", concept);
|
|
6503
|
+
for (const [name, values] of Object.entries(item.facets)) {
|
|
6504
|
+
for (const value of values) pushFacet(result, name, value);
|
|
6505
|
+
}
|
|
6506
|
+
}
|
|
6507
|
+
return result;
|
|
6508
|
+
}
|
|
6509
|
+
function includesAll(actual, expected) {
|
|
6510
|
+
if (!expected || expected.length === 0) return true;
|
|
6511
|
+
const actualSet = new Set(actual);
|
|
6512
|
+
return expected.every((value) => actualSet.has(value));
|
|
6513
|
+
}
|
|
6514
|
+
function matchesFacetFilters(item, filters) {
|
|
6515
|
+
if (!filters) return true;
|
|
6516
|
+
return Object.entries(filters).every(([name, expected]) => includesAll(item.facets[name] ?? [], expected));
|
|
6517
|
+
}
|
|
6518
|
+
function queryAiwgFortemiIndex(index, query = "", options = {}) {
|
|
6519
|
+
const q = query.trim().toLowerCase();
|
|
6520
|
+
const filtered = index.items.filter((item) => {
|
|
6521
|
+
if (q) {
|
|
6522
|
+
const haystack = [item.title, item.text, ...item.tags, ...item.concepts].join("\n").toLowerCase();
|
|
6523
|
+
if (!haystack.includes(q)) return false;
|
|
6524
|
+
}
|
|
6525
|
+
if (options.types && !options.types.includes(item.type)) return false;
|
|
6526
|
+
if (options.privacy && !options.privacy.includes(item.privacy.classification)) return false;
|
|
6527
|
+
if (!includesAll(item.tags, options.tags)) return false;
|
|
6528
|
+
if (!includesAll(item.concepts, options.concepts)) return false;
|
|
6529
|
+
if (!matchesFacetFilters(item, options.facets)) return false;
|
|
6530
|
+
if (options.relationshipTargetId && !item.relationships.some((rel) => rel.target_id === options.relationshipTargetId)) return false;
|
|
6531
|
+
return true;
|
|
6532
|
+
});
|
|
6533
|
+
const offset = options.offset ?? 0;
|
|
6534
|
+
const limit = options.limit ?? filtered.length;
|
|
6535
|
+
return {
|
|
6536
|
+
items: filtered.slice(offset, offset + limit),
|
|
6537
|
+
total: filtered.length,
|
|
6538
|
+
facets: getAiwgFortemiFacets(filtered)
|
|
6539
|
+
};
|
|
6540
|
+
}
|
|
6541
|
+
function createAiwgReviewDecisionExport(source, decisions, generatedAt = (/* @__PURE__ */ new Date()).toISOString()) {
|
|
6542
|
+
return {
|
|
6543
|
+
schema_version: "aiwg.fortemi.review-decisions.v1",
|
|
6544
|
+
generated_at: generatedAt,
|
|
6545
|
+
source_export_schema_version: source.schema_version,
|
|
6546
|
+
decisions: [...decisions].sort((left, right) => left.item_id.localeCompare(right.item_id))
|
|
6547
|
+
};
|
|
6548
|
+
}
|
|
6549
|
+
|
|
5133
6550
|
// src/index.ts
|
|
5134
|
-
var VERSION = "2026.
|
|
6551
|
+
var VERSION = "2026.6.0";
|
|
5135
6552
|
|
|
5136
|
-
export { ArchiveManager, AttachmentsRepository, CURRENT_SHARD_VERSION, CapabilityManager, CaptureKnowledgeInputSchema, CollectionsRepository, FallbackRouter, FortemiToolManifest, GetNoteInputSchema, JOB_CAPABILITIES, JOB_PRIORITIES, JobQueueWorker, LOCAL_ENDPOINTS, LinksRepository, ListNotesInputSchema, ManageArchiveInputSchema, ManageAttachmentsInputSchema, ManageCapabilitiesInputSchema, ManageCollectionsInputSchema, ManageLinksInputSchema, ManageNoteInputSchema, ManageTagsInputSchema, MemoryBlobStore, MigrationRunner, NotesRepository, OpenAICompatibleProvider, PGliteStorageBackend, PGliteStorageBackendFactory, PGliteWorkerClient, ProviderRegistry, SHARD_FORMAT, SearchInputSchema, SearchRepository, SkosRepository, TagsRepository, TransactionProxy, TypedEventBus, VERSION, aiRevisionHandler, allMigrations, appendPluginScript, buildNoteConditions, buildPluginCsp, captureKnowledge, chunkText, classifyError, classifyModel, collectionFromShard, collectionToShard, computeHash, computeSri, conceptTaggingHandler, cosineSimilarity, createBlobStore, createCspReportHandler, createFortemi, createLegacyProvider, createPGliteInstance, createRoutes, defaultStorageBackendFactory, detectGpuCapabilities, detectInferenceCapabilities, discoverLocalProviders, embeddingFromShard, embeddingGenerationHandler, embeddingSetFromShard, embeddingSetMemberToShard, embeddingSetToShard, embeddingToShard, enqueueFullWorkflow, enqueueJob, enqueueNoteCreationJobs, estimateModelFit, estimateVramMB, estimateVramTier, exportShard, fetchPluginScript, fortemiManifest, generateId, getEmbedFunction, getJobQueueStatus, getLlmFunction, getNote, importShard, isPluginScriptAllowed, linkFromShard, linkToShard, linkingHandler, listNotes, manageArchive, manageAttachments, manageCapabilities, manageCollections, manageLinks, manageNote, manageTags, matchRoute, noteFromShard, noteToShard, packTarGz, parseCspReport, registerLlmCapability, registerSemanticCapability, registerServiceWorker, searchTool, selectLlmModel, setEmbedFunction, setLlmFunction, sha256Hex, suggestTags, tagsFromShard, tagsToShard, titleGenerationHandler, unpackTarGz, unregisterLlmCapability, unregisterSemanticCapability, validateChecksums, verifySri };
|
|
6553
|
+
export { ArchiveManager, AttachmentsRepository, CURRENT_SHARD_VERSION, CapabilityManager, CaptureKnowledgeInputSchema, CollectionsRepository, CommunitiesRepository, EmbeddingSetsRepository, FallbackRouter, FortemiToolManifest, GetNoteInputSchema, GraphRepository, JOB_CAPABILITIES, JOB_PRIORITIES, JobQueueWorker, LOCAL_ENDPOINTS, LinksRepository, ListNotesInputSchema, ManageArchiveInputSchema, ManageAttachmentsInputSchema, ManageCapabilitiesInputSchema, ManageCollectionsInputSchema, ManageLinksInputSchema, ManageNoteInputSchema, ManageTagsInputSchema, MemoryBlobStore, MigrationRunner, NotesRepository, OpenAICompatibleProvider, PGliteStorageBackend, PGliteStorageBackendFactory, PGliteWorkerClient, ProviderRegistry, SHARD_FORMAT, SearchInputSchema, SearchRepository, SkosRepository, TagsRepository, TransactionProxy, TypedEventBus, VERSION, aiRevisionHandler, allMigrations, appendPluginScript, assertAiwgFortemiIndexExport, buildNoteConditions, buildPluginCsp, captureKnowledge, chunkText, classifyError, classifyModel, collectionFromShard, collectionToShard, computeHash, computeSri, conceptTaggingHandler, cosineSimilarity, createAiwgReviewDecisionExport, createBlobStore, createCspReportHandler, createFortemi, createLegacyProvider, createPGliteInstance, createRoutes, defaultStorageBackendFactory, detectCommunities, detectGpuCapabilities, detectInferenceCapabilities, discoverLocalProviders, embeddingFromShard, embeddingGenerationHandler, embeddingSetFromShard, embeddingSetMemberToShard, embeddingSetToShard, embeddingToShard, enqueueFullWorkflow, enqueueJob, enqueueNoteCreationJobs, estimateModelFit, estimateVramMB, estimateVramTier, exportShard, fetchPluginScript, fortemiManifest, generateId, getAiwgFortemiFacets, getEmbedFunction, getJobQueueStatus, getLlmFunction, getNote, importShard, isPluginScriptAllowed, linkFromShard, linkToShard, linkingHandler, listNotes, manageArchive, manageAttachments, manageCapabilities, manageCollections, manageLinks, manageNote, manageTags, matchRoute, noteFromShard, noteSkosTagToShard, noteToShard, packTarGz, parseCspReport, provenanceEdgeToShard, queryAiwgFortemiIndex, registerLlmCapability, registerSemanticCapability, registerServiceWorker, searchTool, selectLlmModel, setEmbedFunction, setLlmFunction, sha256Hex, skosConceptToShard, skosRelationToShard, skosSchemeToShard, suggestTags, tagsFromShard, tagsToShard, titleGenerationHandler, unpackTarGz, unregisterLlmCapability, unregisterSemanticCapability, validateAiwgFortemiIndexExport, validateChecksums, verifySri };
|
|
5137
6554
|
//# sourceMappingURL=index.js.map
|
|
5138
6555
|
//# sourceMappingURL=index.js.map
|