@x12i/memorix-service 3.0.2 → 3.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +26 -10
- package/dist/access-gate.d.ts +15 -0
- package/dist/access-gate.d.ts.map +1 -0
- package/dist/access-gate.js +38 -0
- package/dist/access-gate.js.map +1 -0
- package/dist/app.d.ts.map +1 -1
- package/dist/app.js +4 -0
- package/dist/app.js.map +1 -1
- package/dist/callback-grant.d.ts +16 -0
- package/dist/callback-grant.d.ts.map +1 -0
- package/dist/callback-grant.js +72 -0
- package/dist/callback-grant.js.map +1 -0
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/isolation.d.ts +9 -25
- package/dist/isolation.d.ts.map +1 -1
- package/dist/isolation.js +19 -59
- package/dist/isolation.js.map +1 -1
- package/dist/known-packs.d.ts +1 -1
- package/dist/known-packs.d.ts.map +1 -1
- package/dist/known-packs.js +3 -2
- package/dist/known-packs.js.map +1 -1
- package/dist/live.d.ts +8 -0
- package/dist/live.d.ts.map +1 -1
- package/dist/live.js +49 -0
- package/dist/live.js.map +1 -1
- package/dist/openapi/openapi.json +1465 -213
- package/dist/platform.d.ts +20 -1
- package/dist/platform.d.ts.map +1 -1
- package/dist/platform.js +134 -9
- package/dist/platform.js.map +1 -1
- package/dist/relationships/operator-links.d.ts +1 -1
- package/dist/relationships/operator-links.js +1 -1
- package/dist/routes/connector-invocations.d.ts +7 -0
- package/dist/routes/connector-invocations.d.ts.map +1 -0
- package/dist/routes/connector-invocations.js +216 -0
- package/dist/routes/connector-invocations.js.map +1 -0
- package/dist/routes/data.d.ts.map +1 -1
- package/dist/routes/data.js +4 -0
- package/dist/routes/data.js.map +1 -1
- package/dist/routes/memory.d.ts.map +1 -1
- package/dist/routes/memory.js +719 -7
- package/dist/routes/memory.js.map +1 -1
- package/dist/routes/metadata.d.ts.map +1 -1
- package/dist/routes/metadata.js +2 -1
- package/dist/routes/metadata.js.map +1 -1
- package/dist/routes/operations.d.ts.map +1 -1
- package/dist/routes/operations.js +3 -1
- package/dist/routes/operations.js.map +1 -1
- package/dist/routes/pipelines.d.ts.map +1 -1
- package/dist/routes/pipelines.js +67 -0
- package/dist/routes/pipelines.js.map +1 -1
- package/dist/routes/relationships.d.ts.map +1 -1
- package/dist/routes/relationships.js +3 -0
- package/dist/routes/relationships.js.map +1 -1
- package/openapi/openapi.json +1465 -213
- package/package.json +15 -13
package/dist/routes/memory.js
CHANGED
|
@@ -1,5 +1,71 @@
|
|
|
1
|
+
import { cursorKeyFor, listCheckpointSummariesForSource, toCheckpointSummary, resolveCheckpointMethod, upsertMemorySchedule, pullWithSourceClaim, computeNextDueAt, computeInitialNextDueAt, createInMemoryPushEnvelopeStore, } from "@x12i/memorix-memory";
|
|
2
|
+
import { credentialHandleFromDelegation, createCredentialHandle, } from "@x12i/memorix-connector-sdk";
|
|
1
3
|
import { resolveScope } from "../request-scope.js";
|
|
2
4
|
import { ServiceError } from "../errors.js";
|
|
5
|
+
import { forbidCallbackGrant, requireCallbackAccess } from "../access-gate.js";
|
|
6
|
+
const CONNECTION_PROBE_MS = 2000;
|
|
7
|
+
/** Module-level replay ledger — lives for the process lifetime. */
|
|
8
|
+
const pushEnvelopeStore = createInMemoryPushEnvelopeStore({ maxSize: 50_000 });
|
|
9
|
+
/**
|
|
10
|
+
* Resolve credentials for validate/health/configureReceive.
|
|
11
|
+
* Uses a short-lived Credorix delegation when platform.credorix is configured.
|
|
12
|
+
*/
|
|
13
|
+
async function resolveRouteCredentials(platform, scope, source, connectorId, connectorVersion = "0.0.0") {
|
|
14
|
+
const authRef = source.credentialRef ?? null;
|
|
15
|
+
if (!platform.credorix || !authRef) {
|
|
16
|
+
return {
|
|
17
|
+
credentials: createCredentialHandle(authRef, async () => ({})),
|
|
18
|
+
};
|
|
19
|
+
}
|
|
20
|
+
const { provider } = platform.credorix;
|
|
21
|
+
const delegation = await provider.mintDelegation({
|
|
22
|
+
organizationId: scope.orgId,
|
|
23
|
+
sourceId: source.id,
|
|
24
|
+
connectorId,
|
|
25
|
+
connectorVersion,
|
|
26
|
+
authRef,
|
|
27
|
+
purposes: ["provider-http", "webhook-verification"],
|
|
28
|
+
allowedOrigins: ["*"],
|
|
29
|
+
ttlSeconds: 60,
|
|
30
|
+
correlation: { runId: "validate-or-health" },
|
|
31
|
+
});
|
|
32
|
+
return {
|
|
33
|
+
credentials: credentialHandleFromDelegation(provider, delegation.delegationId, authRef),
|
|
34
|
+
invalidate: async () => {
|
|
35
|
+
await provider.invalidate(delegation.delegationId, "validate-or-health-done");
|
|
36
|
+
},
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
function resolveSourceBaseUrl(source) {
|
|
40
|
+
const baseUrlEnv = typeof source.baseUrlEnv === "string" ? source.baseUrlEnv.trim() : "";
|
|
41
|
+
if (baseUrlEnv) {
|
|
42
|
+
const fromEnv = process.env[baseUrlEnv];
|
|
43
|
+
if (fromEnv?.trim())
|
|
44
|
+
return fromEnv.trim().replace(/\/+$/, "");
|
|
45
|
+
}
|
|
46
|
+
const defaultBaseUrl = typeof source.defaultBaseUrl === "string" ? source.defaultBaseUrl.trim() : "";
|
|
47
|
+
if (defaultBaseUrl)
|
|
48
|
+
return defaultBaseUrl.replace(/\/+$/, "");
|
|
49
|
+
return null;
|
|
50
|
+
}
|
|
51
|
+
async function probeSourceHealth(baseUrl) {
|
|
52
|
+
try {
|
|
53
|
+
const res = await fetch(`${baseUrl}/health`, {
|
|
54
|
+
signal: AbortSignal.timeout(CONNECTION_PROBE_MS),
|
|
55
|
+
});
|
|
56
|
+
return res.ok;
|
|
57
|
+
}
|
|
58
|
+
catch {
|
|
59
|
+
return false;
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
function resolveSourceDefinition(platform, agentIds, sourceId) {
|
|
63
|
+
const def = platform.metadata.getDefinition(agentIds, "sources", sourceId);
|
|
64
|
+
if (!def) {
|
|
65
|
+
throw new ServiceError("NOT_FOUND", `source not found: ${sourceId}`, 404);
|
|
66
|
+
}
|
|
67
|
+
return def.definition;
|
|
68
|
+
}
|
|
3
69
|
export async function registerMemoryRoutes(app, platform) {
|
|
4
70
|
app.get("/api/memory/sources", async (req) => {
|
|
5
71
|
const scope = resolveScope(req);
|
|
@@ -9,46 +75,427 @@ export async function registerMemoryRoutes(app, platform) {
|
|
|
9
75
|
.map((e) => {
|
|
10
76
|
const d = e.definition;
|
|
11
77
|
const { credential: _c, credentials: _cs, secret: _s, ...safe } = d;
|
|
12
|
-
return {
|
|
78
|
+
return {
|
|
79
|
+
id: e.id,
|
|
80
|
+
...safe,
|
|
81
|
+
provenance: {
|
|
82
|
+
definedByAgentId: e.provenance.definedByAgentId,
|
|
83
|
+
inheritancePath: e.provenance.inheritancePath,
|
|
84
|
+
selectedAgentId: e.provenance.selectedAgentId,
|
|
85
|
+
},
|
|
86
|
+
};
|
|
13
87
|
});
|
|
14
88
|
return { scope, sources, note: "credentialRef only" };
|
|
15
89
|
});
|
|
90
|
+
app.get("/api/memory/sources/:sourceId/connection", async (req) => {
|
|
91
|
+
const scope = resolveScope(req);
|
|
92
|
+
const sourceId = String(req.params.sourceId ?? "").trim();
|
|
93
|
+
if (!sourceId)
|
|
94
|
+
throw new ServiceError("VALIDATION", "sourceId required", 400);
|
|
95
|
+
const effective = platform.metadata.resolveEffective(scope.agentIds);
|
|
96
|
+
const entry = (effective.effective ?? []).find((e) => e.kind === "sources" && e.id === sourceId);
|
|
97
|
+
if (!entry) {
|
|
98
|
+
throw new ServiceError("NOT_FOUND", `source not found: ${sourceId}`, 404);
|
|
99
|
+
}
|
|
100
|
+
const definition = entry.definition;
|
|
101
|
+
const baseUrl = resolveSourceBaseUrl(definition);
|
|
102
|
+
const checkedAt = new Date().toISOString();
|
|
103
|
+
if (!baseUrl) {
|
|
104
|
+
return {
|
|
105
|
+
scope,
|
|
106
|
+
sourceId,
|
|
107
|
+
status: "Missing URL",
|
|
108
|
+
configured: false,
|
|
109
|
+
reachable: false,
|
|
110
|
+
checkedAt,
|
|
111
|
+
};
|
|
112
|
+
}
|
|
113
|
+
const reachable = await probeSourceHealth(baseUrl);
|
|
114
|
+
return {
|
|
115
|
+
scope,
|
|
116
|
+
sourceId,
|
|
117
|
+
status: reachable ? "Connected" : "Unreachable",
|
|
118
|
+
configured: true,
|
|
119
|
+
reachable,
|
|
120
|
+
checkedAt,
|
|
121
|
+
};
|
|
122
|
+
});
|
|
123
|
+
app.get("/api/memory/sources/:sourceId/checkpoints", async (req) => {
|
|
124
|
+
requireCallbackAccess(req, "checkpoints");
|
|
125
|
+
const scope = resolveScope(req);
|
|
126
|
+
const sourceId = String(req.params.sourceId ?? "").trim();
|
|
127
|
+
if (!sourceId)
|
|
128
|
+
throw new ServiceError("VALIDATION", "sourceId required", 400);
|
|
129
|
+
const source = resolveSourceDefinition(platform, scope.agentIds, sourceId);
|
|
130
|
+
const ports = platform.portsFor(scope);
|
|
131
|
+
const cursors = await ports.memoryRunStore.listCursors({ sourceId });
|
|
132
|
+
const mode = req.query.mode === "context" || req.query.mode === "knowledge"
|
|
133
|
+
? req.query.mode
|
|
134
|
+
: undefined;
|
|
135
|
+
const includeTechnical = req.query.technical === "1" || req.query.technical === "true";
|
|
136
|
+
const checkpoints = listCheckpointSummariesForSource({
|
|
137
|
+
source,
|
|
138
|
+
mode,
|
|
139
|
+
cursors,
|
|
140
|
+
includeTechnical,
|
|
141
|
+
});
|
|
142
|
+
return {
|
|
143
|
+
scope,
|
|
144
|
+
sourceId,
|
|
145
|
+
checkpoints,
|
|
146
|
+
note: "Memorix remembers the last successfully pulled position for every object type populated by this source.",
|
|
147
|
+
};
|
|
148
|
+
});
|
|
149
|
+
app.get("/api/memory/sources/:sourceId/checkpoints/:checkpointKey", async (req) => {
|
|
150
|
+
requireCallbackAccess(req, "checkpoints");
|
|
151
|
+
const scope = resolveScope(req);
|
|
152
|
+
const sourceId = String(req.params.sourceId ?? "").trim();
|
|
153
|
+
const checkpointKey = decodeURIComponent(String(req.params.checkpointKey ?? "").trim());
|
|
154
|
+
if (!sourceId || !checkpointKey) {
|
|
155
|
+
throw new ServiceError("VALIDATION", "sourceId and checkpointKey required", 400);
|
|
156
|
+
}
|
|
157
|
+
resolveSourceDefinition(platform, scope.agentIds, sourceId);
|
|
158
|
+
const ports = platform.portsFor(scope);
|
|
159
|
+
const doc = await ports.memoryRunStore.getCursor(checkpointKey);
|
|
160
|
+
if (!doc || doc.sourceId !== sourceId) {
|
|
161
|
+
throw new ServiceError("NOT_FOUND", `checkpoint not found: ${checkpointKey}`, 404);
|
|
162
|
+
}
|
|
163
|
+
return {
|
|
164
|
+
scope,
|
|
165
|
+
sourceId,
|
|
166
|
+
checkpoint: toCheckpointSummary(doc, { includeTechnical: true }),
|
|
167
|
+
};
|
|
168
|
+
});
|
|
169
|
+
app.post("/api/memory/sources/:sourceId/checkpoints/reset", async (req) => {
|
|
170
|
+
requireCallbackAccess(req, "checkpoints");
|
|
171
|
+
const scope = resolveScope(req);
|
|
172
|
+
const sourceId = String(req.params.sourceId ?? "").trim();
|
|
173
|
+
if (!sourceId)
|
|
174
|
+
throw new ServiceError("VALIDATION", "sourceId required", 400);
|
|
175
|
+
const source = resolveSourceDefinition(platform, scope.agentIds, sourceId);
|
|
176
|
+
const body = (req.body ?? {});
|
|
177
|
+
const ports = platform.portsFor(scope);
|
|
178
|
+
const mode = body.mode ?? "knowledge";
|
|
179
|
+
let reset = [];
|
|
180
|
+
if (body.all) {
|
|
181
|
+
const cursors = await ports.memoryRunStore.listCursors({ sourceId });
|
|
182
|
+
for (const c of cursors) {
|
|
183
|
+
await ports.memoryRunStore.deleteCursor(c.cursorKey);
|
|
184
|
+
reset.push(c.cursorKey);
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
else {
|
|
188
|
+
if (!body.objectType) {
|
|
189
|
+
throw new ServiceError("VALIDATION", "objectType required (or all: true)", 400);
|
|
190
|
+
}
|
|
191
|
+
const route = (source.routes ?? []).find((r) => r.objectType === body.objectType &&
|
|
192
|
+
(body.routeEndpoint == null || r.endpoint === body.routeEndpoint));
|
|
193
|
+
const routeEndpoint = body.routeEndpoint ?? route?.endpoint ?? "";
|
|
194
|
+
const key = cursorKeyFor({
|
|
195
|
+
sourceId,
|
|
196
|
+
objectType: body.objectType,
|
|
197
|
+
mode,
|
|
198
|
+
routeEndpoint,
|
|
199
|
+
workRef: body.workRef,
|
|
200
|
+
});
|
|
201
|
+
const deleted = await ports.memoryRunStore.deleteCursor(key);
|
|
202
|
+
if (deleted)
|
|
203
|
+
reset.push(key);
|
|
204
|
+
}
|
|
205
|
+
return {
|
|
206
|
+
scope,
|
|
207
|
+
sourceId,
|
|
208
|
+
reset,
|
|
209
|
+
note: "Next pull starts from the beginning for the reset checkpoint(s).",
|
|
210
|
+
};
|
|
211
|
+
});
|
|
212
|
+
app.put("/api/memory/sources/:sourceId/checkpoints", async (req) => {
|
|
213
|
+
requireCallbackAccess(req, "checkpoints");
|
|
214
|
+
const scope = resolveScope(req);
|
|
215
|
+
const sourceId = String(req.params.sourceId ?? "").trim();
|
|
216
|
+
if (!sourceId)
|
|
217
|
+
throw new ServiceError("VALIDATION", "sourceId required", 400);
|
|
218
|
+
const source = resolveSourceDefinition(platform, scope.agentIds, sourceId);
|
|
219
|
+
const body = (req.body ?? {});
|
|
220
|
+
if (!body.objectType) {
|
|
221
|
+
throw new ServiceError("VALIDATION", "objectType required", 400);
|
|
222
|
+
}
|
|
223
|
+
const route = (source.routes ?? []).find((r) => r.objectType === body.objectType &&
|
|
224
|
+
(body.routeEndpoint == null || r.endpoint === body.routeEndpoint));
|
|
225
|
+
if (!route && !body.routeEndpoint) {
|
|
226
|
+
throw new ServiceError("VALIDATION", `route not found for objectType ${body.objectType}`, 400);
|
|
227
|
+
}
|
|
228
|
+
const routeEndpoint = body.routeEndpoint ?? route.endpoint;
|
|
229
|
+
const mode = body.mode ?? "knowledge";
|
|
230
|
+
const method = body.method ??
|
|
231
|
+
(route ? resolveCheckpointMethod(route, source) : "cursor");
|
|
232
|
+
const key = cursorKeyFor({
|
|
233
|
+
sourceId,
|
|
234
|
+
objectType: body.objectType,
|
|
235
|
+
mode,
|
|
236
|
+
routeEndpoint,
|
|
237
|
+
workRef: body.workRef,
|
|
238
|
+
});
|
|
239
|
+
const ports = platform.portsFor(scope);
|
|
240
|
+
const existing = await ports.memoryRunStore.getCursor(key);
|
|
241
|
+
const currentRev = existing?.revision ?? 0;
|
|
242
|
+
if (body.expectedRevision != null &&
|
|
243
|
+
body.expectedRevision !== currentRev) {
|
|
244
|
+
throw new ServiceError("CONFLICT", `checkpoint revision conflict: expected ${body.expectedRevision}, got ${currentRev}`, 409);
|
|
245
|
+
}
|
|
246
|
+
const now = new Date().toISOString();
|
|
247
|
+
const doc = {
|
|
248
|
+
cursorKey: key,
|
|
249
|
+
sourceId,
|
|
250
|
+
objectType: body.objectType,
|
|
251
|
+
mode,
|
|
252
|
+
routeEndpoint,
|
|
253
|
+
workRef: body.workRef,
|
|
254
|
+
method,
|
|
255
|
+
committedCursor: body.committedCursor ?? null,
|
|
256
|
+
watermarkValue: body.watermarkValue ?? null,
|
|
257
|
+
fingerprintsEstablished: body.fingerprintsEstablished === true,
|
|
258
|
+
revision: currentRev + 1,
|
|
259
|
+
updatedAt: now,
|
|
260
|
+
lastPullId: body.pullId ?? `svc_${Date.now()}`,
|
|
261
|
+
lastSuccessfulPullAt: now,
|
|
262
|
+
};
|
|
263
|
+
await ports.memoryRunStore.setCursor(doc);
|
|
264
|
+
return {
|
|
265
|
+
scope,
|
|
266
|
+
sourceId,
|
|
267
|
+
checkpoint: toCheckpointSummary(doc, { includeTechnical: true }),
|
|
268
|
+
note: "Administrative checkpoint put is not proof that raw data landed; prefer commit-page for service-controlled collection.",
|
|
269
|
+
};
|
|
270
|
+
});
|
|
16
271
|
app.post("/api/memory/pull", async (req) => {
|
|
272
|
+
forbidCallbackGrant(req);
|
|
17
273
|
const scope = resolveScope(req);
|
|
18
|
-
const body = req.body;
|
|
274
|
+
const body = (req.body ?? {});
|
|
19
275
|
if (body.credential !== undefined) {
|
|
20
276
|
throw new ServiceError("SECRETS_FORBIDDEN", "resolved credentials must not cross the API; use credentialRef only", 400);
|
|
21
277
|
}
|
|
22
278
|
if (!body.sourceId)
|
|
23
279
|
throw new ServiceError("VALIDATION", "sourceId required", 400);
|
|
280
|
+
const mode = body.mode ?? "knowledge";
|
|
24
281
|
const ports = platform.portsFor(scope);
|
|
25
|
-
const
|
|
26
|
-
|
|
282
|
+
const ownerId = `pull:${scope.orgId}:${Date.now()}`;
|
|
283
|
+
const claimed = await pullWithSourceClaim({
|
|
284
|
+
scheduleStore: ports.memoryScheduleStore,
|
|
27
285
|
sourceId: body.sourceId,
|
|
28
|
-
mode
|
|
29
|
-
|
|
286
|
+
mode,
|
|
287
|
+
ownerId,
|
|
288
|
+
pull: () => ports.memory.pull({
|
|
289
|
+
scope,
|
|
290
|
+
sourceId: body.sourceId,
|
|
291
|
+
mode,
|
|
292
|
+
trigger: "on-demand",
|
|
293
|
+
cursor: body.cursor,
|
|
294
|
+
workRef: body.workRef,
|
|
295
|
+
}),
|
|
30
296
|
});
|
|
297
|
+
if ("conflict" in claimed) {
|
|
298
|
+
throw new ServiceError("CONFLICT", `a pull is already in progress for source ${body.sourceId}`, 409);
|
|
299
|
+
}
|
|
300
|
+
const result = claimed.result;
|
|
31
301
|
return {
|
|
32
302
|
scope,
|
|
33
303
|
runId: result.pullId,
|
|
304
|
+
pullId: result.pullId,
|
|
34
305
|
sourceId: body.sourceId,
|
|
35
306
|
mode: result.mode,
|
|
307
|
+
trigger: result.trigger,
|
|
36
308
|
credentialRef: body.credentialRef ?? null,
|
|
37
309
|
status: result.status,
|
|
38
310
|
result: {
|
|
39
311
|
created: result.created,
|
|
40
312
|
updated: result.updated,
|
|
313
|
+
deduplicated: result.deduplicated,
|
|
314
|
+
quarantined: result.quarantined,
|
|
315
|
+
pagesReceived: result.pagesReceived,
|
|
316
|
+
itemsReceived: result.itemsReceived,
|
|
317
|
+
endingCursor: result.endingCursor,
|
|
318
|
+
},
|
|
319
|
+
...(result.failureSummary ? { failureSummary: result.failureSummary } : {}),
|
|
320
|
+
};
|
|
321
|
+
});
|
|
322
|
+
app.get("/api/memory/sources/:sourceId/schedule", async (req) => {
|
|
323
|
+
const scope = resolveScope(req);
|
|
324
|
+
const sourceId = String(req.params.sourceId ?? "").trim();
|
|
325
|
+
if (!sourceId)
|
|
326
|
+
throw new ServiceError("VALIDATION", "sourceId required", 400);
|
|
327
|
+
resolveSourceDefinition(platform, scope.agentIds, sourceId);
|
|
328
|
+
const mode = req.query.mode === "context" ? "context" : "knowledge";
|
|
329
|
+
const ports = platform.portsFor(scope);
|
|
330
|
+
const schedule = await ports.memoryScheduleStore.getSchedule(sourceId, mode);
|
|
331
|
+
return { scope, sourceId, mode, schedule };
|
|
332
|
+
});
|
|
333
|
+
app.put("/api/memory/sources/:sourceId/schedule", async (req) => {
|
|
334
|
+
const scope = resolveScope(req);
|
|
335
|
+
const sourceId = String(req.params.sourceId ?? "").trim();
|
|
336
|
+
if (!sourceId)
|
|
337
|
+
throw new ServiceError("VALIDATION", "sourceId required", 400);
|
|
338
|
+
resolveSourceDefinition(platform, scope.agentIds, sourceId);
|
|
339
|
+
const body = (req.body ?? {});
|
|
340
|
+
if (!body.interval) {
|
|
341
|
+
throw new ServiceError("VALIDATION", "interval required", 400);
|
|
342
|
+
}
|
|
343
|
+
const ports = platform.portsFor(scope);
|
|
344
|
+
try {
|
|
345
|
+
const schedule = await upsertMemorySchedule(ports.memoryScheduleStore, {
|
|
346
|
+
scope,
|
|
347
|
+
sourceId,
|
|
348
|
+
mode: body.mode ?? "knowledge",
|
|
349
|
+
interval: body.interval,
|
|
350
|
+
firstPull: body.firstPull,
|
|
351
|
+
status: body.status,
|
|
352
|
+
});
|
|
353
|
+
return { scope, sourceId, schedule };
|
|
354
|
+
}
|
|
355
|
+
catch (err) {
|
|
356
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
357
|
+
throw new ServiceError("VALIDATION", msg, 400);
|
|
358
|
+
}
|
|
359
|
+
});
|
|
360
|
+
app.post("/api/memory/sources/:sourceId/schedule/pause", async (req) => {
|
|
361
|
+
const scope = resolveScope(req);
|
|
362
|
+
const sourceId = String(req.params.sourceId ?? "").trim();
|
|
363
|
+
if (!sourceId)
|
|
364
|
+
throw new ServiceError("VALIDATION", "sourceId required", 400);
|
|
365
|
+
resolveSourceDefinition(platform, scope.agentIds, sourceId);
|
|
366
|
+
const mode = req.query.mode === "context" ? "context" : "knowledge";
|
|
367
|
+
const ports = platform.portsFor(scope);
|
|
368
|
+
const existing = await ports.memoryScheduleStore.getSchedule(sourceId, mode);
|
|
369
|
+
if (!existing) {
|
|
370
|
+
throw new ServiceError("NOT_FOUND", `schedule not found for ${sourceId}`, 404);
|
|
371
|
+
}
|
|
372
|
+
const schedule = {
|
|
373
|
+
...existing,
|
|
374
|
+
status: "paused",
|
|
375
|
+
updatedAt: new Date().toISOString(),
|
|
376
|
+
revision: existing.revision + 1,
|
|
377
|
+
};
|
|
378
|
+
await ports.memoryScheduleStore.saveSchedule(schedule);
|
|
379
|
+
return { scope, sourceId, schedule };
|
|
380
|
+
});
|
|
381
|
+
app.post("/api/memory/sources/:sourceId/schedule/resume", async (req) => {
|
|
382
|
+
const scope = resolveScope(req);
|
|
383
|
+
const sourceId = String(req.params.sourceId ?? "").trim();
|
|
384
|
+
if (!sourceId)
|
|
385
|
+
throw new ServiceError("VALIDATION", "sourceId required", 400);
|
|
386
|
+
resolveSourceDefinition(platform, scope.agentIds, sourceId);
|
|
387
|
+
const mode = req.query.mode === "context" ? "context" : "knowledge";
|
|
388
|
+
const ports = platform.portsFor(scope);
|
|
389
|
+
const existing = await ports.memoryScheduleStore.getSchedule(sourceId, mode);
|
|
390
|
+
if (!existing) {
|
|
391
|
+
throw new ServiceError("NOT_FOUND", `schedule not found for ${sourceId}`, 404);
|
|
392
|
+
}
|
|
393
|
+
const now = new Date();
|
|
394
|
+
let nextDueAt = existing.nextDueAt;
|
|
395
|
+
if (nextDueAt <= now.toISOString()) {
|
|
396
|
+
nextDueAt = computeInitialNextDueAt(now, existing.interval, "immediate");
|
|
397
|
+
}
|
|
398
|
+
const schedule = {
|
|
399
|
+
...existing,
|
|
400
|
+
status: "active",
|
|
401
|
+
nextDueAt,
|
|
402
|
+
updatedAt: now.toISOString(),
|
|
403
|
+
revision: existing.revision + 1,
|
|
404
|
+
};
|
|
405
|
+
await ports.memoryScheduleStore.saveSchedule(schedule);
|
|
406
|
+
return { scope, sourceId, schedule };
|
|
407
|
+
});
|
|
408
|
+
app.post("/api/memory/sources/:sourceId/schedule/run-now", async (req) => {
|
|
409
|
+
const scope = resolveScope(req);
|
|
410
|
+
const sourceId = String(req.params.sourceId ?? "").trim();
|
|
411
|
+
if (!sourceId)
|
|
412
|
+
throw new ServiceError("VALIDATION", "sourceId required", 400);
|
|
413
|
+
resolveSourceDefinition(platform, scope.agentIds, sourceId);
|
|
414
|
+
const mode = req.query.mode === "context" ? "context" : "knowledge";
|
|
415
|
+
const ports = platform.portsFor(scope);
|
|
416
|
+
const existing = await ports.memoryScheduleStore.getSchedule(sourceId, mode);
|
|
417
|
+
const ownerId = `run-now:${scope.orgId}:${Date.now()}`;
|
|
418
|
+
const claimed = await pullWithSourceClaim({
|
|
419
|
+
scheduleStore: ports.memoryScheduleStore,
|
|
420
|
+
sourceId,
|
|
421
|
+
mode,
|
|
422
|
+
ownerId,
|
|
423
|
+
pull: () => ports.memory.pull({
|
|
424
|
+
scope,
|
|
425
|
+
sourceId,
|
|
426
|
+
mode,
|
|
427
|
+
trigger: "scheduled",
|
|
428
|
+
}),
|
|
429
|
+
});
|
|
430
|
+
if ("conflict" in claimed) {
|
|
431
|
+
throw new ServiceError("CONFLICT", `a pull is already in progress for source ${sourceId}`, 409);
|
|
432
|
+
}
|
|
433
|
+
const result = claimed.result;
|
|
434
|
+
const now = new Date();
|
|
435
|
+
if (existing) {
|
|
436
|
+
await ports.memoryScheduleStore.saveSchedule({
|
|
437
|
+
...existing,
|
|
438
|
+
lastFiredAt: now.toISOString(),
|
|
439
|
+
lastPullId: result.pullId,
|
|
440
|
+
lastStatus: result.status,
|
|
441
|
+
nextDueAt: computeNextDueAt(now, existing.interval),
|
|
442
|
+
updatedAt: now.toISOString(),
|
|
443
|
+
revision: existing.revision + 1,
|
|
444
|
+
});
|
|
445
|
+
}
|
|
446
|
+
return {
|
|
447
|
+
scope,
|
|
448
|
+
sourceId,
|
|
449
|
+
mode,
|
|
450
|
+
pullId: result.pullId,
|
|
451
|
+
status: result.status,
|
|
452
|
+
result: {
|
|
453
|
+
created: result.created,
|
|
454
|
+
updated: result.updated,
|
|
455
|
+
deduplicated: result.deduplicated,
|
|
41
456
|
quarantined: result.quarantined,
|
|
42
457
|
pagesReceived: result.pagesReceived,
|
|
458
|
+
itemsReceived: result.itemsReceived,
|
|
43
459
|
endingCursor: result.endingCursor,
|
|
44
460
|
},
|
|
461
|
+
schedule: existing
|
|
462
|
+
? await ports.memoryScheduleStore.getSchedule(sourceId, mode)
|
|
463
|
+
: null,
|
|
45
464
|
};
|
|
46
465
|
});
|
|
466
|
+
app.delete("/api/memory/sources/:sourceId/schedule", async (req) => {
|
|
467
|
+
const scope = resolveScope(req);
|
|
468
|
+
const sourceId = String(req.params.sourceId ?? "").trim();
|
|
469
|
+
if (!sourceId)
|
|
470
|
+
throw new ServiceError("VALIDATION", "sourceId required", 400);
|
|
471
|
+
resolveSourceDefinition(platform, scope.agentIds, sourceId);
|
|
472
|
+
const mode = req.query.mode === "context" ? "context" : "knowledge";
|
|
473
|
+
const ports = platform.portsFor(scope);
|
|
474
|
+
const removed = await ports.memoryScheduleStore.deleteSchedule(sourceId, mode);
|
|
475
|
+
return { scope, sourceId, mode, removed };
|
|
476
|
+
});
|
|
47
477
|
app.get("/api/memory/runs", async (req) => {
|
|
48
478
|
const scope = resolveScope(req);
|
|
49
|
-
|
|
479
|
+
const q = req.query;
|
|
480
|
+
const ports = platform.portsFor(scope);
|
|
481
|
+
const limit = q.limit ? Number(q.limit) : 50;
|
|
482
|
+
const runs = await ports.memoryRunStore.listRuns({
|
|
483
|
+
sourceId: q.sourceId,
|
|
484
|
+
mode: q.mode,
|
|
485
|
+
limit: Number.isFinite(limit) ? limit : 50,
|
|
486
|
+
});
|
|
487
|
+
let quarantine = await ports.memoryRunStore.listQuarantine();
|
|
488
|
+
if (q.sourceId) {
|
|
489
|
+
quarantine = quarantine.filter((item) => item.sourceId === q.sourceId);
|
|
490
|
+
}
|
|
491
|
+
const cursors = await ports.memoryRunStore.listCursors({
|
|
492
|
+
sourceId: q.sourceId,
|
|
493
|
+
});
|
|
494
|
+
const checkpoints = cursors.map((c) => toCheckpointSummary(c));
|
|
495
|
+
return { scope, runs, quarantine, checkpoints };
|
|
50
496
|
});
|
|
51
497
|
app.get("/api/memory/raw", async (req) => {
|
|
498
|
+
requireCallbackAccess(req, "raw");
|
|
52
499
|
const scope = resolveScope(req);
|
|
53
500
|
const q = req.query;
|
|
54
501
|
if (!q.objectType || !q.id) {
|
|
@@ -76,5 +523,270 @@ export async function registerMemoryRoutes(app, platform) {
|
|
|
76
523
|
note: "exact raw .data landing; no secrets; associated* must not be copied to root",
|
|
77
524
|
};
|
|
78
525
|
});
|
|
526
|
+
app.post("/api/memory/sources/:sourceId/commit-page", async (req) => {
|
|
527
|
+
requireCallbackAccess(req, "checkpoints");
|
|
528
|
+
requireCallbackAccess(req, "raw");
|
|
529
|
+
const scope = resolveScope(req);
|
|
530
|
+
const sourceId = String(req.params.sourceId ?? "").trim();
|
|
531
|
+
if (!sourceId)
|
|
532
|
+
throw new ServiceError("VALIDATION", "sourceId required", 400);
|
|
533
|
+
const source = resolveSourceDefinition(platform, scope.agentIds, sourceId);
|
|
534
|
+
const body = (req.body ?? {});
|
|
535
|
+
if (!body.objectType) {
|
|
536
|
+
throw new ServiceError("VALIDATION", "objectType required", 400);
|
|
537
|
+
}
|
|
538
|
+
if (!Array.isArray(body.items)) {
|
|
539
|
+
throw new ServiceError("VALIDATION", "items array required", 400);
|
|
540
|
+
}
|
|
541
|
+
if (body.expectedCheckpointRevision == null) {
|
|
542
|
+
throw new ServiceError("VALIDATION", "expectedCheckpointRevision required", 400);
|
|
543
|
+
}
|
|
544
|
+
const route = (source.routes ?? []).find((r) => r.objectType === body.objectType &&
|
|
545
|
+
(body.routeEndpoint == null || r.endpoint === body.routeEndpoint));
|
|
546
|
+
const streamMatch = typeof body.streamId === "string" && body.streamId.trim()
|
|
547
|
+
? (source.streams ?? []).find((s) => s.streamId === body.streamId)
|
|
548
|
+
: undefined;
|
|
549
|
+
if (!route && !streamMatch) {
|
|
550
|
+
throw new ServiceError("VALIDATION", `route not found for objectType ${body.objectType}`, 400);
|
|
551
|
+
}
|
|
552
|
+
const ports = platform.portsFor(scope);
|
|
553
|
+
let receipt;
|
|
554
|
+
try {
|
|
555
|
+
receipt = await ports.memory.commitPage({
|
|
556
|
+
scope,
|
|
557
|
+
sourceId,
|
|
558
|
+
objectType: body.objectType,
|
|
559
|
+
routeEndpoint: body.routeEndpoint ??
|
|
560
|
+
route?.endpoint ??
|
|
561
|
+
streamMatch?.endpoint ??
|
|
562
|
+
"",
|
|
563
|
+
mode: body.mode ?? "knowledge",
|
|
564
|
+
workRef: body.workRef,
|
|
565
|
+
items: body.items,
|
|
566
|
+
nextCheckpoint: body.nextCheckpoint ?? {},
|
|
567
|
+
expectedCheckpointRevision: body.expectedCheckpointRevision,
|
|
568
|
+
pullId: body.pullId,
|
|
569
|
+
streamId: body.streamId,
|
|
570
|
+
connectorId: body.connectorId,
|
|
571
|
+
connectorVersion: body.connectorVersion,
|
|
572
|
+
trigger: body.trigger,
|
|
573
|
+
});
|
|
574
|
+
}
|
|
575
|
+
catch (err) {
|
|
576
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
577
|
+
if (/CHECKPOINT_REVISION_CONFLICT/i.test(msg)) {
|
|
578
|
+
throw new ServiceError("CONFLICT", msg, 409);
|
|
579
|
+
}
|
|
580
|
+
throw new ServiceError("MEMORY_COMMIT_FAILED", msg, 400);
|
|
581
|
+
}
|
|
582
|
+
return { scope, sourceId, ...receipt };
|
|
583
|
+
});
|
|
584
|
+
/** Provider-originated push ingress — same land/quarantine/run path as pulls (FR-024). */
|
|
585
|
+
app.post("/api/memory/sources/:sourceId/receive", async (req) => {
|
|
586
|
+
forbidCallbackGrant(req);
|
|
587
|
+
const scope = resolveScope(req);
|
|
588
|
+
const sourceId = String(req.params.sourceId ?? "").trim();
|
|
589
|
+
if (!sourceId)
|
|
590
|
+
throw new ServiceError("VALIDATION", "sourceId required", 400);
|
|
591
|
+
resolveSourceDefinition(platform, scope.agentIds, sourceId);
|
|
592
|
+
const body = (req.body ?? {});
|
|
593
|
+
// Delivery replay guard — short-circuit duplicate provider deliveries.
|
|
594
|
+
const deliveryId = body.deliveryId?.trim() || null;
|
|
595
|
+
if (deliveryId) {
|
|
596
|
+
const prior = await pushEnvelopeStore.getByDeliveryId(deliveryId, sourceId);
|
|
597
|
+
if (prior?.result) {
|
|
598
|
+
return {
|
|
599
|
+
scope,
|
|
600
|
+
sourceId,
|
|
601
|
+
result: prior.result,
|
|
602
|
+
replayed: true,
|
|
603
|
+
deliveryId,
|
|
604
|
+
note: "Duplicate delivery detected; returning cached result.",
|
|
605
|
+
};
|
|
606
|
+
}
|
|
607
|
+
}
|
|
608
|
+
const ports = platform.portsFor(scope);
|
|
609
|
+
const result = await ports.memory.receive({
|
|
610
|
+
scope,
|
|
611
|
+
sourceId,
|
|
612
|
+
mode: body.mode ?? "knowledge",
|
|
613
|
+
trigger: "provider_push",
|
|
614
|
+
workRef: body.workRef,
|
|
615
|
+
receivePayload: body.payload ?? req.body,
|
|
616
|
+
receiveHeaders: {
|
|
617
|
+
...(body.headers ?? {}),
|
|
618
|
+
...(deliveryId ? { "x-delivery-id": deliveryId } : {}),
|
|
619
|
+
},
|
|
620
|
+
});
|
|
621
|
+
// Persist delivery so future duplicates can be short-circuited.
|
|
622
|
+
if (deliveryId) {
|
|
623
|
+
await pushEnvelopeStore.save({
|
|
624
|
+
deliveryId,
|
|
625
|
+
receivedAt: new Date().toISOString(),
|
|
626
|
+
sourceId,
|
|
627
|
+
method: req.method,
|
|
628
|
+
path: req.url,
|
|
629
|
+
result: {
|
|
630
|
+
status: result.status,
|
|
631
|
+
created: result.created ?? 0,
|
|
632
|
+
updated: result.updated ?? 0,
|
|
633
|
+
deduplicated: result.deduplicated ?? 0,
|
|
634
|
+
quarantined: result.quarantined ?? 0,
|
|
635
|
+
},
|
|
636
|
+
});
|
|
637
|
+
}
|
|
638
|
+
return { scope, sourceId, result, ...(deliveryId ? { deliveryId } : {}) };
|
|
639
|
+
});
|
|
640
|
+
app.post("/api/memory/sources/:sourceId/validate", async (req) => {
|
|
641
|
+
forbidCallbackGrant(req);
|
|
642
|
+
const scope = resolveScope(req);
|
|
643
|
+
const sourceId = String(req.params.sourceId ?? "").trim();
|
|
644
|
+
if (!sourceId)
|
|
645
|
+
throw new ServiceError("VALIDATION", "sourceId required", 400);
|
|
646
|
+
const source = resolveSourceDefinition(platform, scope.agentIds, sourceId);
|
|
647
|
+
const connectorRef = source.connectorRef
|
|
648
|
+
?? source.kind;
|
|
649
|
+
const connector = connectorRef
|
|
650
|
+
? platform.connectorRegistry.resolveRuntime(connectorRef)
|
|
651
|
+
: null;
|
|
652
|
+
if (connector?.validate) {
|
|
653
|
+
const { credentials, invalidate } = await resolveRouteCredentials(platform, scope, source, connectorRef ?? "unknown", connector.manifest?.version ?? "0.0.0");
|
|
654
|
+
try {
|
|
655
|
+
const result = await connector.validate({
|
|
656
|
+
source,
|
|
657
|
+
credentials,
|
|
658
|
+
});
|
|
659
|
+
return { scope, sourceId, ...result };
|
|
660
|
+
}
|
|
661
|
+
catch (err) {
|
|
662
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
663
|
+
return { scope, sourceId, ok: false, checks: [], error: msg };
|
|
664
|
+
}
|
|
665
|
+
finally {
|
|
666
|
+
await invalidate?.();
|
|
667
|
+
}
|
|
668
|
+
}
|
|
669
|
+
return {
|
|
670
|
+
scope,
|
|
671
|
+
sourceId,
|
|
672
|
+
ok: true,
|
|
673
|
+
checks: [
|
|
674
|
+
{
|
|
675
|
+
code: "SOURCE_RESOLVED",
|
|
676
|
+
status: "pass",
|
|
677
|
+
message: `source ${source.id} resolved for connector ${connectorRef ?? source.kind}`,
|
|
678
|
+
},
|
|
679
|
+
{
|
|
680
|
+
code: "CREDENTIAL_REF",
|
|
681
|
+
status: source.credentialRef ? "pass" : "warning",
|
|
682
|
+
message: source.credentialRef
|
|
683
|
+
? `credentialRef ${source.credentialRef} present`
|
|
684
|
+
: "no credentialRef configured",
|
|
685
|
+
},
|
|
686
|
+
],
|
|
687
|
+
note: "Connector.validate hooks are invoked when a framework connector declares validation capability.",
|
|
688
|
+
};
|
|
689
|
+
});
|
|
690
|
+
app.get("/api/memory/sources/:sourceId/health", async (req) => {
|
|
691
|
+
const scope = resolveScope(req);
|
|
692
|
+
const sourceId = String(req.params.sourceId ?? "").trim();
|
|
693
|
+
if (!sourceId)
|
|
694
|
+
throw new ServiceError("VALIDATION", "sourceId required", 400);
|
|
695
|
+
const source = resolveSourceDefinition(platform, scope.agentIds, sourceId);
|
|
696
|
+
const connectorRef = source.connectorRef
|
|
697
|
+
?? source.kind;
|
|
698
|
+
const connector = connectorRef
|
|
699
|
+
? platform.connectorRegistry.resolveRuntime(connectorRef)
|
|
700
|
+
: null;
|
|
701
|
+
if (connector?.health) {
|
|
702
|
+
const { credentials, invalidate } = await resolveRouteCredentials(platform, scope, source, connectorRef ?? "unknown", connector.manifest?.version ?? "0.0.0");
|
|
703
|
+
try {
|
|
704
|
+
const result = await connector.health({
|
|
705
|
+
source,
|
|
706
|
+
credentials,
|
|
707
|
+
});
|
|
708
|
+
return {
|
|
709
|
+
scope, sourceId,
|
|
710
|
+
...result,
|
|
711
|
+
checks: result.checks ?? [],
|
|
712
|
+
};
|
|
713
|
+
}
|
|
714
|
+
catch (err) {
|
|
715
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
716
|
+
return { scope, sourceId, ok: false, checks: [], error: msg };
|
|
717
|
+
}
|
|
718
|
+
finally {
|
|
719
|
+
await invalidate?.();
|
|
720
|
+
}
|
|
721
|
+
}
|
|
722
|
+
return {
|
|
723
|
+
scope,
|
|
724
|
+
sourceId,
|
|
725
|
+
ok: true,
|
|
726
|
+
checks: [],
|
|
727
|
+
detail: "connector host reachable (framework health hook optional)",
|
|
728
|
+
};
|
|
729
|
+
});
|
|
730
|
+
app.get("/api/connectors", async (req) => {
|
|
731
|
+
const scope = resolveScope(req);
|
|
732
|
+
const frameworkConnectors = platform.connectorRegistry.list().map((doc) => ({
|
|
733
|
+
id: doc.connectorId,
|
|
734
|
+
version: doc.version,
|
|
735
|
+
protocol: doc.protocol,
|
|
736
|
+
enabled: doc.enabled,
|
|
737
|
+
workflowHash: doc.workflowHash,
|
|
738
|
+
}));
|
|
739
|
+
const legacyConnectors = [
|
|
740
|
+
{ id: "rest-fixture", version: "0.0.0-legacy", protocol: "memorix-connector/1", enabled: true },
|
|
741
|
+
{ id: "opx-fixture", version: "0.0.0-legacy", protocol: "memorix-connector/1", enabled: true },
|
|
742
|
+
{ id: "knowx-fixture", version: "0.0.0-legacy", protocol: "memorix-connector/1", enabled: true },
|
|
743
|
+
];
|
|
744
|
+
// Deduplicate: framework connectors take precedence over legacy stubs.
|
|
745
|
+
const frameworkIds = new Set(frameworkConnectors.map((c) => c.id));
|
|
746
|
+
const connectors = [
|
|
747
|
+
...frameworkConnectors,
|
|
748
|
+
...legacyConnectors.filter((c) => !frameworkIds.has(c.id)),
|
|
749
|
+
];
|
|
750
|
+
return { scope, connectors };
|
|
751
|
+
});
|
|
752
|
+
/** Admin-only subscription management — never mixed into Pull (FR-028). */
|
|
753
|
+
app.post("/api/memory/sources/:sourceId/receive:configure", async (req) => {
|
|
754
|
+
forbidCallbackGrant(req);
|
|
755
|
+
const scope = resolveScope(req);
|
|
756
|
+
const sourceId = String(req.params.sourceId ?? "").trim();
|
|
757
|
+
if (!sourceId)
|
|
758
|
+
throw new ServiceError("VALIDATION", "sourceId required", 400);
|
|
759
|
+
const source = resolveSourceDefinition(platform, scope.agentIds, sourceId);
|
|
760
|
+
const body = (req.body ?? {});
|
|
761
|
+
const action = body.action ?? "create";
|
|
762
|
+
const connectorRef = source.connectorRef ??
|
|
763
|
+
source.kind;
|
|
764
|
+
const connector = connectorRef
|
|
765
|
+
? platform.connectorRegistry.resolveRuntime(connectorRef)
|
|
766
|
+
: null;
|
|
767
|
+
if (!connector?.configureReceive) {
|
|
768
|
+
return {
|
|
769
|
+
scope,
|
|
770
|
+
sourceId,
|
|
771
|
+
ok: false,
|
|
772
|
+
audited: true,
|
|
773
|
+
action,
|
|
774
|
+
note: "connector does not declare configureReceive",
|
|
775
|
+
};
|
|
776
|
+
}
|
|
777
|
+
const { credentials, invalidate } = await resolveRouteCredentials(platform, scope, source, connectorRef ?? "unknown", connector.manifest?.version ?? "0.0.0");
|
|
778
|
+
try {
|
|
779
|
+
const result = await connector.configureReceive({
|
|
780
|
+
source,
|
|
781
|
+
action,
|
|
782
|
+
subscription: body.subscription,
|
|
783
|
+
credentials,
|
|
784
|
+
});
|
|
785
|
+
return { scope, sourceId, ok: true, audited: true, action, result };
|
|
786
|
+
}
|
|
787
|
+
finally {
|
|
788
|
+
await invalidate?.();
|
|
789
|
+
}
|
|
790
|
+
});
|
|
79
791
|
}
|
|
80
792
|
//# sourceMappingURL=memory.js.map
|