archgraph-argo 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/README.md +115 -0
- package/argo/package.json +8 -0
- package/argo/rules/intent-architecture-global-rule.md +45 -0
- package/argo/schema/ImplementationToCodingHandoff.schema.json +252 -0
- package/argo/schema/ImplementationToIntentTraceProposal.schema.json +180 -0
- package/argo/schema/IntentToImplementationHandoff.schema.json +75 -0
- package/argo/schema/SystemArchitecture.schema.json +378 -0
- package/argo/schema/archimate3.2.pdf +0 -0
- package/argo/scripts/ARCHITECTURE.md +57 -0
- package/argo/scripts/archimate32-rules.js +12301 -0
- package/argo/scripts/argo-mcp-server.js +629 -0
- package/argo/scripts/argo-paths.js +77 -0
- package/argo/scripts/ensureArgoHarnessEnvironment.js +340 -0
- package/argo/scripts/generateArchitectureDiffPlantuml.js +466 -0
- package/argo/scripts/graph-rag/ARCHITECTURE.md +192 -0
- package/argo/scripts/graph-rag/canonicalProjectionAuthority.js +45 -0
- package/argo/scripts/graph-rag/defaultSemanticRetrieval.js +969 -0
- package/argo/scripts/graph-rag/embeddingQualificationGate.js +59 -0
- package/argo/scripts/graph-rag/externalProductionConfig.js +74 -0
- package/argo/scripts/graph-rag/liveEmbeddingIndexGate.js +129 -0
- package/argo/scripts/graph-rag/liveEmbeddingNeo4jBoundary.js +137 -0
- package/argo/scripts/graph-rag/liveEmbeddingProviderClient.js +49 -0
- package/argo/scripts/graph-rag/liveEmbeddingProviderConfig.js +481 -0
- package/argo/scripts/graph-rag/mutationEmbeddingVectorLifecycle.js +1261 -0
- package/argo/scripts/graph-rag/neo4jNativeRetrieval.js +37 -0
- package/argo/scripts/graph-rag/productionGraphRagRuntime.js +1624 -0
- package/argo/scripts/graph-rag/semantic-persistence/ARCHITECTURE.md +51 -0
- package/argo/scripts/graph-rag/semantic-persistence/productionSemanticBackfill.js +241 -0
- package/argo/scripts/graph-rag/semantic-persistence/productionSemanticCheckpointStore.js +99 -0
- package/argo/scripts/graph-rag/semantic-persistence/productionSemanticNeo4jAdapter.js +149 -0
- package/argo/scripts/graph-rag/semantic-persistence/productionSemanticProjectionStore.js +171 -0
- package/argo/scripts/graph-rag/semanticOperatorError.js +38 -0
- package/argo/scripts/graph-rag/semanticOperatorJourney.js +459 -0
- package/argo/scripts/graph-rag/semanticReadinessAttestationStore.js +398 -0
- package/argo/scripts/graph-rag/systemMetadataCommandAdapter.js +269 -0
- package/argo/scripts/graph-semantics.js +220 -0
- package/argo/scripts/neo4j-system-architecture-store.js +777 -0
- package/argo/scripts/repositoryArgoEnvironment.js +101 -0
- package/argo/scripts/runArchitectureTests.js +583 -0
- package/argo/scripts/semanticOperatorJourneyCli.js +91 -0
- package/argo/scripts/syncSystemArchitectureToNeo4j.js +67 -0
- package/argo/scripts/systemarchitecture-mcp-server.js +2965 -0
- package/argo/scripts/test-executors/_template.js +58 -0
- package/argo/scripts/test-executors/default.js +199 -0
- package/argo/scripts/validateStageHandoff.js +459 -0
- package/argo/scripts/validateSystemArchitecture.js +254 -0
- package/argo/scripts/validateTraceProposal.js +181 -0
- package/argo/scripts/validator-mcp-server.js +377 -0
- package/argo/skills/argo-init/SKILL.md +110 -0
- package/bin/argo-deploy.js +12 -0
- package/install-argo.ps1 +112 -0
- package/package.json +28 -0
- package/vendor/neo4j-driver-6.2.0.tgz +0 -0
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
# Production Semantic Persistence Contract
|
|
2
|
+
|
|
3
|
+
This local contract refines `OVERALL_ARCHITECTURE.md` and the parent `.argo/scripts/graph-rag/ARCHITECTURE.md`.
|
|
4
|
+
|
|
5
|
+
## Responsibilities
|
|
6
|
+
|
|
7
|
+
- `productionSemanticBackfill.js` owns the explicit WP-P1 backfill use case through `createProductionSemanticBackfill(dependencies).execute({ explicitOptIn })`.
|
|
8
|
+
- Backfill begins only after the injected structural-projection boundary proves the same canonical version complete. Structural projection is a prerequisite, not semantic readiness.
|
|
9
|
+
- Backfill reads one immutable canonical snapshot and independently enumerates every Element, ArchitectureRelationship, and View. It never fabricates a canonical mutation to trigger embedding generation.
|
|
10
|
+
- Work is bounded by an explicit positive batch size. Every channel has durable totals, completed counts, cursor, canonical version, retry, and isolated-failure checkpoints.
|
|
11
|
+
- Resume starts from durable channel checkpoints and does not repeat completed work. Acceptance independently compares provider and durable-upsert canonical identities before and after interruption; an implementation-reported resume flag is not evidence. Rerun performs stable-identity upserts and is idempotent for unchanged canonical/content/index/provider/model/version/dimensions/vector evidence.
|
|
12
|
+
- A record failure is observable and isolated from other records. Partial or failed channels remain non-Aligned. Alignment may become `Aligned` only after Element, ArchitectureRelationship, and View channels all report complete for the same canonical version.
|
|
13
|
+
- `productionSemanticProjectionStore.js` owns `createProductionSemanticProjectionStore(dependencies)` and exposes only `upsertRecords(records)`, `deleteTombstones(tombstones)`, `readRecords()`, and `close()`.
|
|
14
|
+
- `productionSemanticNeo4jAdapter.js` owns `createProductionSemanticNeo4jAdapter(dependencies)` as the concrete durable production projection adapter. `productionSemanticCheckpointStore.js` owns `createProductionSemanticCheckpointStore(dependencies)` as the concrete durable per-channel checkpoint adapter. Deterministic tests inject only a recording raw Neo4j driver beneath these production factories; they do not substitute an in-memory projection or checkpoint implementation.
|
|
15
|
+
- The production store validates stable canonical identity and complete channel, canonical/content/index, provider/model/model-version/dimensions, and vector metadata before persistence.
|
|
16
|
+
- The production store depends inward on a durable Neo4j persistence adapter. It must MERGE/upsert changed records by stable canonical identity and delete tombstones by that identity.
|
|
17
|
+
- Production records have no `runId`; a `runId`-bearing record is rejected before persistence. The store's complete callable public surface is exactly `upsertRecords`, `deleteTombstones`, `readRecords`, and `close`: no cleanup, runId-delete, truncate, clear, or reset API is permitted. Production never imports or delegates to `liveEmbeddingNeo4jBoundary.js`.
|
|
18
|
+
- Existing live E2E runId cleanup remains test-only and unchanged. It cannot select or delete production semantic projection labels or identities.
|
|
19
|
+
- The same four-method production store is the only durable target for automatic incremental indexing. Successful batch and focused Element, ArchitectureRelationship, and View add/update/remove writes provide exact touched identities: add/update records use `upsertRecords`, removed identities use `deleteTombstones`, and unrelated records remain unchanged across restart. Incremental records carry the same complete canonical/content/index/provider/model/model-version/dimensions/vector contract as full reconciliation.
|
|
20
|
+
- Readiness invalidation belongs to the canonical-write orchestration boundary and occurs before this store or provider is called. This directory may report persistence success/failure but cannot mark Aligned; queryability and global coherence are later outer checks.
|
|
21
|
+
- External Neo4j and provider credentials use the existing approved external configuration and qualification boundaries. Missing configuration or provider qualification blocks before provider, projection, checkpoint, or index side effects. Missing `neo4jUri` blocks startup; tests never invent synchronization evidence.
|
|
22
|
+
- Canonical JSON remains authority at `design/KG/SystemArchitecture.json`. The durable Neo4j records are subordinate projection/index state and have no API that writes canonical JSON.
|
|
23
|
+
|
|
24
|
+
## Public interface
|
|
25
|
+
|
|
26
|
+
- `createProductionSemanticBackfill(dependencies)` requires canonical-source, structural-projection, qualified-provider, durable projection-store, checkpoint-store, external configuration, qualification, and bounded-batch dependencies; it returns `execute({ explicitOptIn })`.
|
|
27
|
+
- `createProductionSemanticProjectionStore(dependencies)` requires the concrete durable adapter, canonical-authority policy, external configuration, and qualified-provider evidence; it returns exactly the four store methods listed above.
|
|
28
|
+
- `createProductionSemanticNeo4jAdapter(dependencies)` and `createProductionSemanticCheckpointStore(dependencies)` translate store/checkpoint operations to the configured durable Neo4j driver boundary.
|
|
29
|
+
- The parent runtime exposes `runSemanticBackfill(request)` and composes the concrete adapter, store, checkpoint store, and backfill from `semanticPersistence` production dependencies.
|
|
30
|
+
- The parent durable incremental lifecycle composes the existing projection store directly for exact touched upsert/tombstone operations. It does not reuse live-E2E `writeEvidence(runId, ...)` or cleanup APIs.
|
|
31
|
+
- Canonical argo init privately delegates to `runtime.runSemanticBackfill(request)` after its exact enabled/valid gate decision. No standalone MCP backfill route is exposed. The private composition must exist without Harness injection; missing internal consent fails `SP01_EXPLICIT_OPT_IN_REQUIRED`, and structural/canonical mismatch fails `SP01_STRUCTURAL_VERSION_MISMATCH`, both before provider or durable effects. Structural mutation uses the incremental lifecycle and never fakes a full-backfill trigger.
|
|
32
|
+
|
|
33
|
+
## Local dependencies
|
|
34
|
+
|
|
35
|
+
- MCP gateway → production Graph RAG composition → semantic backfill → canonical/structural/provider/store/checkpoint ports.
|
|
36
|
+
- Semantic backfill may depend on provider and projection interfaces; provider and store adapters must not depend outward on backfill orchestration.
|
|
37
|
+
- Production semantic persistence may depend on the approved Neo4j JavaScript driver adapter and existing external configuration/qualification modules.
|
|
38
|
+
- No file in this directory may depend on `tests/`, Python, Neo4j GenAI procedures, the live-E2E evidence boundary, or mutable canonical-write internals.
|
|
39
|
+
- Checkpoint persistence and semantic projection persistence are durable production state. Neither is a process-local map in production composition.
|
|
40
|
+
|
|
41
|
+
## Owned tests
|
|
42
|
+
|
|
43
|
+
- `tests/harness/productionSemanticPersistenceHarness.js`
|
|
44
|
+
- `tests/explicit/entries/runProductionSemanticBackfill.js` — SP-01 control point: shipped non-injected JSON-RPC `tools/call` plus explicit backfill after structural projection. Observation: the default MCP path owns production composition and fails closed before secrets/Neo4j, while the deterministic injected raw-driver scenario proves independent complete channels, bounded checkpoints, interruption/resume, isolated failure, idempotent rerun, complete metadata, no fake mutation, and alignment only after all channels complete.
|
|
45
|
+
- `tests/explicit/entries/runPersistentSemanticProjectionLifecycle.js` — SP-02 control point: durable projection across restart, changed-record upsert, tombstone deletion, and unrelated live-E2E cleanup. Observation: stable identity and metadata survive, production has no runId cleanup, canonical authority remains intact, and Neo4j remains subordinate.
|
|
46
|
+
- `tests/architecture/production-semantic-persistence/architecture-boundary.guard.js` — `ArchitectureBoundaryGuard`.
|
|
47
|
+
- `tests/architecture/production-semantic-persistence/dependency-direction.guard.js` — `DependencyDirectionGuard`.
|
|
48
|
+
- `tests/architecture/production-semantic-persistence/explicit-entrypoint-correctness.guard.js` — `ExplicitEntrypointCorrectnessGuard`.
|
|
49
|
+
- `tests/architecture/production-semantic-persistence/implementation-traceability.guard.js` — `KeyImplementationTraceabilityGuard`.
|
|
50
|
+
|
|
51
|
+
All owned Harness, explicit entrypoints, guards, this contract, the root and parent contracts, incoming intent handoff, runner failure records, and canonical graph are frozen during Coding/Repair. The protected fixture is `canonicalThreeChannelFixture`; the protected baseline is the committed WP-P1 pre-coding full-run result in `design/KG/test-failure-records.json`.
|
|
@@ -0,0 +1,241 @@
|
|
|
1
|
+
const crypto = require('node:crypto');
|
|
2
|
+
const {
|
|
3
|
+
evaluateEmbeddingQualification,
|
|
4
|
+
} = require('../embeddingQualificationGate.js');
|
|
5
|
+
|
|
6
|
+
const CHANNELS = Object.freeze(['Element', 'ArchitectureRelationship', 'View']);
|
|
7
|
+
const CHANNEL_SOURCES = Object.freeze({
|
|
8
|
+
Element: Object.freeze({ property: 'elements', identity: record => record.id }),
|
|
9
|
+
ArchitectureRelationship: Object.freeze({ property: 'relationships', identity: record => record.id }),
|
|
10
|
+
View: Object.freeze({ property: 'views', identity: record => record.view_id }),
|
|
11
|
+
});
|
|
12
|
+
|
|
13
|
+
function createProductionSemanticBackfill(dependencies = {}) {
|
|
14
|
+
requireBoundary(dependencies.canonicalSource, 'readSnapshot', 'canonicalSource');
|
|
15
|
+
requireBoundary(dependencies.structuralProjection, 'requireComplete', 'structuralProjection');
|
|
16
|
+
requireBoundary(dependencies.embeddingProvider, 'embedBatch', 'embeddingProvider');
|
|
17
|
+
requireBoundary(dependencies.projectionStore, 'upsertRecords', 'projectionStore');
|
|
18
|
+
requireBoundary(dependencies.checkpointStore, 'readCheckpoint', 'checkpointStore');
|
|
19
|
+
requireBoundary(dependencies.checkpointStore, 'writeCheckpoint', 'checkpointStore');
|
|
20
|
+
const batchSize = dependencies.batchSize;
|
|
21
|
+
if (!Number.isInteger(batchSize) || batchSize <= 0) {
|
|
22
|
+
throw categoryError('SP01_POSITIVE_BATCH_SIZE_REQUIRED', 'batchSize');
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
return Object.freeze({
|
|
26
|
+
async execute(request = {}) {
|
|
27
|
+
if (request.explicitOptIn !== true) {
|
|
28
|
+
throw categoryError('SP01_EXPLICIT_OPT_IN_REQUIRED', 'explicitOptIn');
|
|
29
|
+
}
|
|
30
|
+
requireExternalConfiguration(dependencies.configuration);
|
|
31
|
+
const qualification = evaluateEmbeddingQualification(dependencies.qualification);
|
|
32
|
+
|
|
33
|
+
const structural = await dependencies.structuralProjection.requireComplete();
|
|
34
|
+
const snapshot = await dependencies.canonicalSource.readSnapshot();
|
|
35
|
+
const canonicalVersion = snapshot && snapshot.version;
|
|
36
|
+
if (
|
|
37
|
+
!structural
|
|
38
|
+
|| structural.status !== 'complete'
|
|
39
|
+
|| !isNonBlankString(structural.canonicalVersion)
|
|
40
|
+
|| structural.canonicalVersion !== canonicalVersion
|
|
41
|
+
) {
|
|
42
|
+
throw categoryError('SP01_STRUCTURAL_VERSION_MISMATCH', 'canonicalVersion');
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
const channels = {};
|
|
46
|
+
for (const channel of CHANNELS) {
|
|
47
|
+
channels[channel] = await processChannel({
|
|
48
|
+
channel,
|
|
49
|
+
snapshot,
|
|
50
|
+
canonicalVersion,
|
|
51
|
+
qualification,
|
|
52
|
+
batchSize,
|
|
53
|
+
embeddingProvider: dependencies.embeddingProvider,
|
|
54
|
+
projectionStore: dependencies.projectionStore,
|
|
55
|
+
checkpointStore: dependencies.checkpointStore,
|
|
56
|
+
});
|
|
57
|
+
}
|
|
58
|
+
const aligned = CHANNELS.every(channel => (
|
|
59
|
+
channels[channel].status === 'complete'
|
|
60
|
+
&& channels[channel].canonicalVersion === canonicalVersion
|
|
61
|
+
));
|
|
62
|
+
return Object.freeze({
|
|
63
|
+
status: aligned ? 'passed' : 'partial',
|
|
64
|
+
canonicalVersion,
|
|
65
|
+
alignmentState: aligned ? 'Aligned' : 'Updating',
|
|
66
|
+
channels: Object.freeze(channels),
|
|
67
|
+
});
|
|
68
|
+
},
|
|
69
|
+
});
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
async function processChannel(options) {
|
|
73
|
+
const source = CHANNEL_SOURCES[options.channel];
|
|
74
|
+
const canonicalRecords = (options.snapshot[source.property] || []).map(record => Object.freeze({
|
|
75
|
+
canonicalIdentity: `${options.channel}:${source.identity(record)}`,
|
|
76
|
+
channel: options.channel,
|
|
77
|
+
canonicalObject: record,
|
|
78
|
+
}));
|
|
79
|
+
const stored = await options.checkpointStore.readCheckpoint(options.channel);
|
|
80
|
+
const checkpoint = stored && stored.canonicalVersion === options.canonicalVersion
|
|
81
|
+
? mutableCheckpoint(stored, canonicalRecords.length)
|
|
82
|
+
: emptyCheckpoint(options.channel, options.canonicalVersion, canonicalRecords.length);
|
|
83
|
+
const attempts = new Map();
|
|
84
|
+
|
|
85
|
+
while (checkpoint.completedCanonicalIdentities.length < canonicalRecords.length) {
|
|
86
|
+
const completed = new Set(checkpoint.completedCanonicalIdentities);
|
|
87
|
+
const pending = canonicalRecords.filter(record => (
|
|
88
|
+
!completed.has(record.canonicalIdentity)
|
|
89
|
+
&& (attempts.get(record.canonicalIdentity) || 0) < 2
|
|
90
|
+
));
|
|
91
|
+
if (pending.length === 0) {
|
|
92
|
+
break;
|
|
93
|
+
}
|
|
94
|
+
const batch = pending.slice(0, options.batchSize);
|
|
95
|
+
for (const record of batch) {
|
|
96
|
+
attempts.set(record.canonicalIdentity, (attempts.get(record.canonicalIdentity) || 0) + 1);
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
const providerResult = await options.embeddingProvider.embedBatch(Object.freeze(batch));
|
|
100
|
+
const vectors = new Map((providerResult && providerResult.vectors ? providerResult.vectors : [])
|
|
101
|
+
.map(result => [result.canonicalIdentity, result.vector]));
|
|
102
|
+
const providerFailures = providerResult && Array.isArray(providerResult.failures)
|
|
103
|
+
? providerResult.failures
|
|
104
|
+
: [];
|
|
105
|
+
const successfulRecords = [];
|
|
106
|
+
const currentFailures = [];
|
|
107
|
+
|
|
108
|
+
for (const record of batch) {
|
|
109
|
+
const vector = vectors.get(record.canonicalIdentity);
|
|
110
|
+
if (Array.isArray(vector)) {
|
|
111
|
+
successfulRecords.push(buildSemanticRecord(record, vector, options));
|
|
112
|
+
completed.add(record.canonicalIdentity);
|
|
113
|
+
continue;
|
|
114
|
+
}
|
|
115
|
+
const observed = providerFailures.find(failure => failure.canonicalIdentity === record.canonicalIdentity);
|
|
116
|
+
currentFailures.push(Object.freeze({
|
|
117
|
+
canonicalIdentity: record.canonicalIdentity,
|
|
118
|
+
category: observed && observed.category ? observed.category : 'PROVIDER_RECORD_FAILED',
|
|
119
|
+
}));
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
if (successfulRecords.length > 0) {
|
|
123
|
+
await options.projectionStore.upsertRecords(successfulRecords);
|
|
124
|
+
}
|
|
125
|
+
checkpoint.completedCanonicalIdentities = [...completed];
|
|
126
|
+
checkpoint.completedCount = completed.size;
|
|
127
|
+
checkpoint.cursor = completed.size;
|
|
128
|
+
checkpoint.retries += currentFailures.length;
|
|
129
|
+
checkpoint.isolatedFailures = mergeFailures(checkpoint.isolatedFailures, currentFailures);
|
|
130
|
+
checkpoint.status = completed.size === canonicalRecords.length ? 'complete' : 'partial';
|
|
131
|
+
await options.checkpointStore.writeCheckpoint(freezeCheckpoint(checkpoint));
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
return Object.freeze({
|
|
135
|
+
status: checkpoint.completedCanonicalIdentities.length === canonicalRecords.length ? 'complete' : 'partial',
|
|
136
|
+
canonicalVersion: options.canonicalVersion,
|
|
137
|
+
total: canonicalRecords.length,
|
|
138
|
+
completedCount: checkpoint.completedCanonicalIdentities.length,
|
|
139
|
+
cursor: checkpoint.cursor,
|
|
140
|
+
retries: checkpoint.retries,
|
|
141
|
+
isolatedFailures: Object.freeze(checkpoint.isolatedFailures.map(failure => Object.freeze({ ...failure }))),
|
|
142
|
+
});
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
function buildSemanticRecord(record, vector, options) {
|
|
146
|
+
const contentHash = crypto.createHash('sha256')
|
|
147
|
+
.update(JSON.stringify(record.canonicalObject))
|
|
148
|
+
.digest('hex');
|
|
149
|
+
return Object.freeze({
|
|
150
|
+
canonicalIdentity: record.canonicalIdentity,
|
|
151
|
+
channel: record.channel,
|
|
152
|
+
canonicalVersion: options.canonicalVersion,
|
|
153
|
+
contentVersion: `content:${contentHash}`,
|
|
154
|
+
indexVersion: `index:${contentHash}`,
|
|
155
|
+
provider: options.qualification.provider,
|
|
156
|
+
model: options.qualification.model,
|
|
157
|
+
modelVersion: options.qualification.version,
|
|
158
|
+
dimensions: options.qualification.dimensions,
|
|
159
|
+
vector: Object.freeze([...vector]),
|
|
160
|
+
});
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
function emptyCheckpoint(channel, canonicalVersion, total) {
|
|
164
|
+
return {
|
|
165
|
+
channel,
|
|
166
|
+
canonicalVersion,
|
|
167
|
+
total,
|
|
168
|
+
completedCount: 0,
|
|
169
|
+
cursor: 0,
|
|
170
|
+
retries: 0,
|
|
171
|
+
completedCanonicalIdentities: [],
|
|
172
|
+
isolatedFailures: [],
|
|
173
|
+
status: total === 0 ? 'complete' : 'pending',
|
|
174
|
+
};
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
function mutableCheckpoint(checkpoint, total) {
|
|
178
|
+
return {
|
|
179
|
+
channel: checkpoint.channel,
|
|
180
|
+
canonicalVersion: checkpoint.canonicalVersion,
|
|
181
|
+
total,
|
|
182
|
+
completedCount: checkpoint.completedCount || 0,
|
|
183
|
+
cursor: checkpoint.cursor || 0,
|
|
184
|
+
retries: checkpoint.retries || 0,
|
|
185
|
+
completedCanonicalIdentities: [...(checkpoint.completedCanonicalIdentities || [])],
|
|
186
|
+
isolatedFailures: [...(checkpoint.isolatedFailures || [])],
|
|
187
|
+
status: checkpoint.status || 'pending',
|
|
188
|
+
};
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
function freezeCheckpoint(checkpoint) {
|
|
192
|
+
return Object.freeze({
|
|
193
|
+
...checkpoint,
|
|
194
|
+
completedCanonicalIdentities: Object.freeze([...checkpoint.completedCanonicalIdentities]),
|
|
195
|
+
isolatedFailures: Object.freeze(checkpoint.isolatedFailures.map(failure => Object.freeze({ ...failure }))),
|
|
196
|
+
});
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
function mergeFailures(existing, current) {
|
|
200
|
+
const merged = new Map(existing.map(failure => [failure.canonicalIdentity, failure]));
|
|
201
|
+
for (const failure of current) {
|
|
202
|
+
merged.set(failure.canonicalIdentity, failure);
|
|
203
|
+
}
|
|
204
|
+
return [...merged.values()];
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
function requireBoundary(boundary, method, name) {
|
|
208
|
+
if (!boundary || typeof boundary[method] !== 'function') {
|
|
209
|
+
throw new TypeError(`${name}.${method} is required`);
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
function requireExternalConfiguration(configuration) {
|
|
214
|
+
const supplied = configuration && typeof configuration === 'object' ? configuration : {};
|
|
215
|
+
const required = [
|
|
216
|
+
['neo4jDatabaseUrl', 'neo4jUri'],
|
|
217
|
+
['neo4jDatabaseUsername', 'neo4jUsername'],
|
|
218
|
+
['neo4jDatabasePassword', 'neo4jPassword'],
|
|
219
|
+
['embeddingCredential'],
|
|
220
|
+
];
|
|
221
|
+
for (const alternatives of required) {
|
|
222
|
+
if (alternatives.map(field => supplied[field]).find(isNonBlankString) === undefined) {
|
|
223
|
+
throw categoryError('EXTERNAL_CREDENTIALS_REQUIRED', alternatives[0]);
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
function isNonBlankString(value) {
|
|
229
|
+
return typeof value === 'string' && value.trim() !== '';
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
function categoryError(category, field) {
|
|
233
|
+
const error = new Error(category);
|
|
234
|
+
error.category = category;
|
|
235
|
+
error.field = field;
|
|
236
|
+
return error;
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
module.exports = {
|
|
240
|
+
createProductionSemanticBackfill,
|
|
241
|
+
};
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
const CHECKPOINT_LABEL = 'ArgoProductionSemanticCheckpoint';
|
|
2
|
+
|
|
3
|
+
function createProductionSemanticCheckpointStore(dependencies = {}) {
|
|
4
|
+
const { driver } = dependencies;
|
|
5
|
+
if (!driver || (typeof driver.execute !== 'function' && typeof driver.session !== 'function')) {
|
|
6
|
+
throw new TypeError('driver.execute or driver.session is required');
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
return Object.freeze({
|
|
10
|
+
async readCheckpoint(channel) {
|
|
11
|
+
if (typeof driver.execute === 'function') {
|
|
12
|
+
const result = await driver.execute(Object.freeze({
|
|
13
|
+
kind: 'semantic-checkpoint-read',
|
|
14
|
+
channel,
|
|
15
|
+
}));
|
|
16
|
+
return result && result.checkpoint ? cloneCheckpoint(result.checkpoint) : undefined;
|
|
17
|
+
}
|
|
18
|
+
return withSession(driver, dependencies.configuration, async session => {
|
|
19
|
+
const result = await executeRead(
|
|
20
|
+
session,
|
|
21
|
+
`MATCH (checkpoint:${CHECKPOINT_LABEL} {channel: $channel}) RETURN checkpoint.payload AS payload`,
|
|
22
|
+
{ channel },
|
|
23
|
+
);
|
|
24
|
+
const record = result.records && result.records[0];
|
|
25
|
+
const payload = record && typeof record.get === 'function' ? record.get('payload') : record && record.payload;
|
|
26
|
+
return typeof payload === 'string' ? cloneCheckpoint(JSON.parse(payload)) : undefined;
|
|
27
|
+
});
|
|
28
|
+
},
|
|
29
|
+
|
|
30
|
+
async writeCheckpoint(checkpoint) {
|
|
31
|
+
const durableCheckpoint = cloneCheckpoint(checkpoint);
|
|
32
|
+
if (typeof driver.execute === 'function') {
|
|
33
|
+
const result = await driver.execute(Object.freeze({
|
|
34
|
+
kind: 'semantic-checkpoint-write',
|
|
35
|
+
checkpoint: durableCheckpoint,
|
|
36
|
+
}));
|
|
37
|
+
return result && result.checkpoint ? cloneCheckpoint(result.checkpoint) : durableCheckpoint;
|
|
38
|
+
}
|
|
39
|
+
await withSession(driver, dependencies.configuration, session => executeWrite(
|
|
40
|
+
session,
|
|
41
|
+
[
|
|
42
|
+
`MERGE (checkpoint:${CHECKPOINT_LABEL} {channel: $channel})`,
|
|
43
|
+
'SET checkpoint.canonicalVersion = $canonicalVersion, checkpoint.payload = $payload',
|
|
44
|
+
'RETURN checkpoint.payload AS payload',
|
|
45
|
+
].join('\n'),
|
|
46
|
+
{
|
|
47
|
+
channel: durableCheckpoint.channel,
|
|
48
|
+
canonicalVersion: durableCheckpoint.canonicalVersion,
|
|
49
|
+
payload: JSON.stringify(durableCheckpoint),
|
|
50
|
+
},
|
|
51
|
+
));
|
|
52
|
+
return durableCheckpoint;
|
|
53
|
+
},
|
|
54
|
+
|
|
55
|
+
async close() {
|
|
56
|
+
if (typeof driver.close === 'function') {
|
|
57
|
+
await driver.close();
|
|
58
|
+
}
|
|
59
|
+
},
|
|
60
|
+
});
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
async function withSession(driver, configuration, action) {
|
|
64
|
+
const database = configuration && configuration.neo4jDatabase;
|
|
65
|
+
const session = driver.session(database === undefined ? undefined : { database });
|
|
66
|
+
try {
|
|
67
|
+
return await action(session);
|
|
68
|
+
} finally {
|
|
69
|
+
if (session && typeof session.close === 'function') {
|
|
70
|
+
await session.close();
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
function executeWrite(session, query, parameters) {
|
|
76
|
+
if (typeof session.executeWrite === 'function') {
|
|
77
|
+
return session.executeWrite(transaction => transaction.run(query, parameters));
|
|
78
|
+
}
|
|
79
|
+
return session.run(query, parameters);
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
function executeRead(session, query, parameters) {
|
|
83
|
+
if (typeof session.executeRead === 'function') {
|
|
84
|
+
return session.executeRead(transaction => transaction.run(query, parameters));
|
|
85
|
+
}
|
|
86
|
+
return session.run(query, parameters);
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
function cloneCheckpoint(checkpoint) {
|
|
90
|
+
return Object.freeze({
|
|
91
|
+
...checkpoint,
|
|
92
|
+
completedCanonicalIdentities: Object.freeze([...(checkpoint.completedCanonicalIdentities || [])]),
|
|
93
|
+
isolatedFailures: Object.freeze((checkpoint.isolatedFailures || []).map(failure => Object.freeze({ ...failure }))),
|
|
94
|
+
});
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
module.exports = {
|
|
98
|
+
createProductionSemanticCheckpointStore,
|
|
99
|
+
};
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
const SEMANTIC_LABEL = 'ArgoProductionSemanticRecord';
|
|
2
|
+
const CHANNEL_INDEXES = Object.freeze({
|
|
3
|
+
Element: Object.freeze({
|
|
4
|
+
label: 'ArgoProductionSemanticElement',
|
|
5
|
+
indexName: 'argo_production_semantic_element_vector',
|
|
6
|
+
}),
|
|
7
|
+
ArchitectureRelationship: Object.freeze({
|
|
8
|
+
label: 'ArgoProductionSemanticRelationship',
|
|
9
|
+
indexName: 'argo_production_semantic_relationship_vector',
|
|
10
|
+
}),
|
|
11
|
+
View: Object.freeze({
|
|
12
|
+
label: 'ArgoProductionSemanticView',
|
|
13
|
+
indexName: 'argo_production_semantic_view_vector',
|
|
14
|
+
}),
|
|
15
|
+
});
|
|
16
|
+
|
|
17
|
+
function createProductionSemanticNeo4jAdapter(dependencies = {}) {
|
|
18
|
+
const { driver } = dependencies;
|
|
19
|
+
if (!driver || (typeof driver.execute !== 'function' && typeof driver.session !== 'function')) {
|
|
20
|
+
throw new TypeError('driver.execute or driver.session is required');
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
return Object.freeze({
|
|
24
|
+
async upsertRecords(records) {
|
|
25
|
+
if (typeof driver.execute === 'function') {
|
|
26
|
+
return driver.execute(Object.freeze({
|
|
27
|
+
kind: 'semantic-record-upsert',
|
|
28
|
+
records: Object.freeze(records.map(cloneRecord)),
|
|
29
|
+
}));
|
|
30
|
+
}
|
|
31
|
+
return withSession(driver, dependencies.configuration, async session => {
|
|
32
|
+
await ensureVectorIndexes(session);
|
|
33
|
+
const results = [];
|
|
34
|
+
for (const [channel, definition] of Object.entries(CHANNEL_INDEXES)) {
|
|
35
|
+
const channelRecords = records.filter(record => record.channel === channel).map(cloneRecord);
|
|
36
|
+
if (channelRecords.length === 0) continue;
|
|
37
|
+
const query = [
|
|
38
|
+
'UNWIND $records AS record',
|
|
39
|
+
`MERGE (semantic:${SEMANTIC_LABEL} {canonicalIdentity: record.canonicalIdentity})`,
|
|
40
|
+
'SET semantic = record',
|
|
41
|
+
`SET semantic:${definition.label}`,
|
|
42
|
+
'RETURN count(semantic) AS count',
|
|
43
|
+
].join('\n');
|
|
44
|
+
results.push(await executeWrite(session, query, { records: channelRecords }));
|
|
45
|
+
}
|
|
46
|
+
return results[results.length - 1] || { records: [] };
|
|
47
|
+
});
|
|
48
|
+
},
|
|
49
|
+
|
|
50
|
+
async deleteTombstones(tombstones) {
|
|
51
|
+
if (typeof driver.execute === 'function') {
|
|
52
|
+
return driver.execute(Object.freeze({
|
|
53
|
+
kind: 'semantic-record-delete-tombstones',
|
|
54
|
+
tombstones: Object.freeze(tombstones.map(cloneRecord)),
|
|
55
|
+
}));
|
|
56
|
+
}
|
|
57
|
+
return withSession(driver, dependencies.configuration, async session => {
|
|
58
|
+
const query = [
|
|
59
|
+
'UNWIND $canonicalIdentities AS canonicalIdentity',
|
|
60
|
+
`MATCH (semantic:${SEMANTIC_LABEL} {canonicalIdentity: canonicalIdentity})`,
|
|
61
|
+
'DETACH DELETE semantic',
|
|
62
|
+
'RETURN count(*) AS count',
|
|
63
|
+
].join('\n');
|
|
64
|
+
return executeWrite(session, query, {
|
|
65
|
+
canonicalIdentities: tombstones.map(item => item.canonicalIdentity),
|
|
66
|
+
});
|
|
67
|
+
});
|
|
68
|
+
},
|
|
69
|
+
|
|
70
|
+
async readRecords() {
|
|
71
|
+
if (typeof driver.execute === 'function') {
|
|
72
|
+
const result = await driver.execute(Object.freeze({ kind: 'semantic-record-read-all' }));
|
|
73
|
+
return Object.freeze((result && result.records ? result.records : []).map(cloneRecord));
|
|
74
|
+
}
|
|
75
|
+
return withSession(driver, dependencies.configuration, async session => {
|
|
76
|
+
const result = await executeRead(
|
|
77
|
+
session,
|
|
78
|
+
`MATCH (semantic:${SEMANTIC_LABEL}) RETURN properties(semantic) AS record ORDER BY semantic.canonicalIdentity`,
|
|
79
|
+
{},
|
|
80
|
+
);
|
|
81
|
+
return Object.freeze((result.records || []).map(resultRecord).map(cloneRecord));
|
|
82
|
+
});
|
|
83
|
+
},
|
|
84
|
+
|
|
85
|
+
async close() {
|
|
86
|
+
if (typeof driver.close === 'function') {
|
|
87
|
+
await driver.close();
|
|
88
|
+
}
|
|
89
|
+
},
|
|
90
|
+
});
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
async function ensureVectorIndexes(session) {
|
|
94
|
+
for (const definition of Object.values(CHANNEL_INDEXES)) {
|
|
95
|
+
await executeWrite(
|
|
96
|
+
session,
|
|
97
|
+
[
|
|
98
|
+
`CREATE VECTOR INDEX ${definition.indexName} IF NOT EXISTS`,
|
|
99
|
+
`FOR (semantic:${definition.label}) ON (semantic.vector)`,
|
|
100
|
+
'OPTIONS { indexConfig: { `vector.dimensions`: 1024, `vector.similarity_function`: "cosine" } }',
|
|
101
|
+
].join('\n'),
|
|
102
|
+
{},
|
|
103
|
+
);
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
async function withSession(driver, configuration, action) {
|
|
108
|
+
const database = configuration && configuration.neo4jDatabase;
|
|
109
|
+
const session = driver.session(database === undefined ? undefined : { database });
|
|
110
|
+
try {
|
|
111
|
+
return await action(session);
|
|
112
|
+
} finally {
|
|
113
|
+
if (session && typeof session.close === 'function') {
|
|
114
|
+
await session.close();
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
function executeWrite(session, query, parameters) {
|
|
120
|
+
if (typeof session.executeWrite === 'function') {
|
|
121
|
+
return session.executeWrite(transaction => transaction.run(query, parameters));
|
|
122
|
+
}
|
|
123
|
+
return session.run(query, parameters);
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
function executeRead(session, query, parameters) {
|
|
127
|
+
if (typeof session.executeRead === 'function') {
|
|
128
|
+
return session.executeRead(transaction => transaction.run(query, parameters));
|
|
129
|
+
}
|
|
130
|
+
return session.run(query, parameters);
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
function resultRecord(record) {
|
|
134
|
+
if (record && typeof record.get === 'function') {
|
|
135
|
+
return record.get('record');
|
|
136
|
+
}
|
|
137
|
+
return record && record.record ? record.record : record;
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
function cloneRecord(record) {
|
|
141
|
+
return Object.freeze({
|
|
142
|
+
...record,
|
|
143
|
+
...(Array.isArray(record && record.vector) ? { vector: Object.freeze([...record.vector]) } : {}),
|
|
144
|
+
});
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
module.exports = {
|
|
148
|
+
createProductionSemanticNeo4jAdapter,
|
|
149
|
+
};
|