@remnic/core 9.15.2 → 9.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/access-admin-ops-surface.d.ts +8 -8
- package/dist/access-admin-ops-surface.js +7 -7
- package/dist/access-boundary.d.ts +8 -8
- package/dist/access-boundary.js +7 -7
- package/dist/access-cli.js +31 -30
- package/dist/access-cli.js.map +1 -1
- package/dist/access-http.d.ts +8 -8
- package/dist/access-http.js +11 -11
- package/dist/access-identity-continuity-surface.d.ts +6 -6
- package/dist/access-identity-continuity-surface.js +7 -7
- package/dist/access-lcm-surface.d.ts +8 -8
- package/dist/access-lcm-surface.js +7 -7
- package/dist/access-mcp.d.ts +8 -8
- package/dist/access-mcp.js +10 -10
- package/dist/access-memory-search-fanout.d.ts +2 -2
- package/dist/{access-namespace-preflight-B9e5GnyZ.d.ts → access-namespace-preflight-DwhEFYy3.d.ts} +1 -1
- package/dist/access-namespace-preflight.d.ts +3 -3
- package/dist/access-namespace-preflight.js +7 -7
- package/dist/access-observe-write-surface.d.ts +8 -8
- package/dist/access-observe-write-surface.js +7 -7
- package/dist/access-operations-batch.js +8 -8
- package/dist/access-operations.d.ts +11 -11
- package/dist/access-operations.js +9 -9
- package/dist/access-recall-concurrency.d.ts +8 -8
- package/dist/access-recall-concurrency.js +7 -7
- package/dist/access-recall-response.d.ts +8 -8
- package/dist/access-recall-response.js +7 -7
- package/dist/access-recall-surface.d.ts +8 -8
- package/dist/access-recall-surface.js +7 -7
- package/dist/access-schema.d.ts +76 -76
- package/dist/{access-service-DgmJd_A-.d.ts → access-service-CSq1Wp2Z.d.ts} +6 -6
- package/dist/access-service.d.ts +8 -8
- package/dist/access-service.js +7 -7
- package/dist/access-surface-catalog.d.ts +8 -8
- package/dist/action-confidence.d.ts +1 -1
- package/dist/active-memory-bridge.d.ts +1 -1
- package/dist/active-recall.d.ts +1 -1
- package/dist/active-recall.js +2 -1
- package/dist/active-recall.js.map +1 -1
- package/dist/behavior-learner.d.ts +1 -1
- package/dist/behavior-signals.d.ts +1 -1
- package/dist/bootstrap.d.ts +6 -6
- package/dist/briefing.d.ts +2 -2
- package/dist/briefing.js +3 -3
- package/dist/buffer-surprise-report.d.ts +1 -1
- package/dist/buffer.d.ts +2 -2
- package/dist/bulk-import/index.d.ts +3 -3
- package/dist/calibration.d.ts +1 -1
- package/dist/capabilities.d.ts +1 -1
- package/dist/{catalog-g_dfT6oL.d.ts → catalog-B9uga4sG.d.ts} +1 -1
- package/dist/causal-behavior.d.ts +1 -1
- package/dist/causal-consolidation.d.ts +1 -1
- package/dist/causal-consolidation.js +4 -4
- package/dist/causal-trajectory-graph.d.ts +1 -1
- package/dist/{chunk-IRX5UCGN.js → chunk-2OH3ZRYH.js} +3 -3
- package/dist/{chunk-YXTSUIXC.js → chunk-5EKOBRCA.js} +6 -1
- package/dist/chunk-5EKOBRCA.js.map +1 -0
- package/dist/{chunk-42DJZZUB.js → chunk-7LXSPIVG.js} +2 -2
- package/dist/{chunk-47WFEKT7.js → chunk-7PK4W4RN.js} +2 -2
- package/dist/{chunk-O7ACJAHR.js → chunk-7YJLCNPF.js} +2 -2
- package/dist/{chunk-6EJL2O3G.js → chunk-AFH7PIFU.js} +2 -2
- package/dist/{chunk-JN4N2TMQ.js → chunk-ANGUOE2N.js} +7 -7
- package/dist/{chunk-GQOKO77V.js → chunk-AWT7SCWQ.js} +2 -2
- package/dist/{chunk-PXJEL55X.js → chunk-BIRPR5N7.js} +2 -2
- package/dist/{chunk-NAY25BGS.js → chunk-BTK2PIFL.js} +6 -6
- package/dist/{chunk-SYCVU556.js → chunk-C2RMQRPN.js} +3 -3
- package/dist/{chunk-T4EAQPQQ.js → chunk-G2WCVFDC.js} +2 -2
- package/dist/{chunk-UJIBXOJA.js → chunk-GEDBTSYU.js} +5 -5
- package/dist/{chunk-CXFVZ3PK.js → chunk-GRJ5UOFO.js} +84 -658
- package/dist/chunk-GRJ5UOFO.js.map +1 -0
- package/dist/chunk-GSQASASV.js +429 -0
- package/dist/chunk-GSQASASV.js.map +1 -0
- package/dist/{chunk-WIKOHCQZ.js → chunk-I6ODVWQB.js} +2 -2
- package/dist/{chunk-H4MDUPER.js → chunk-IZHXSLO7.js} +2 -2
- package/dist/{chunk-RRS5CDB6.js → chunk-JOCXI2O5.js} +2 -2
- package/dist/{chunk-EEUZQQRJ.js → chunk-KGKYTCDO.js} +2 -2
- package/dist/{chunk-JTW4TGPV.js → chunk-L3HAUO7K.js} +2 -2
- package/dist/{chunk-P53HGVJS.js → chunk-LMVAIZR5.js} +2 -2
- package/dist/{chunk-KLXG4BBO.js → chunk-M6TZQDTX.js} +2 -2
- package/dist/{chunk-FJ5IQ3SI.js → chunk-N7DK67GZ.js} +1 -1
- package/dist/chunk-N7DK67GZ.js.map +1 -0
- package/dist/{chunk-U6QWUFSM.js → chunk-OXT5TQDS.js} +2 -2
- package/dist/{chunk-T4OTDMX4.js → chunk-QDTEH6BS.js} +2 -2
- package/dist/{chunk-5MDE5MIC.js → chunk-SESF64QR.js} +2 -2
- package/dist/{chunk-4E5H4FPC.js → chunk-TUNI4GB3.js} +3 -3
- package/dist/{chunk-XJT6LNLY.js → chunk-V6LYAXGW.js} +834 -25
- package/dist/chunk-V6LYAXGW.js.map +1 -0
- package/dist/{chunk-5HFIHWOF.js → chunk-VLIY55Q5.js} +17 -17
- package/dist/{chunk-BNH2VVWP.js → chunk-WIS5JTJR.js} +2 -2
- package/dist/{chunk-OOGGUXAG.js → chunk-WQ2I6ODB.js} +2 -2
- package/dist/{cli-BXCd2cgK.d.ts → cli-DYLGXxuu.d.ts} +5 -5
- package/dist/cli.d.ts +10 -10
- package/dist/cli.js +25 -24
- package/dist/compounding/engine.d.ts +2 -2
- package/dist/compounding/engine.js +3 -3
- package/dist/compounding/preference-consolidator.d.ts +1 -1
- package/dist/compression-optimizer.d.ts +1 -1
- package/dist/config.d.ts +1 -1
- package/dist/config.js +2 -1
- package/dist/connectors/codex-materialize-runner.d.ts +1 -1
- package/dist/connectors/codex-materialize-runner.js +3 -3
- package/dist/connectors/codex-materialize.d.ts +1 -1
- package/dist/connectors/index.d.ts +1 -1
- package/dist/connectors/index.js +3 -3
- package/dist/consolidation-provenance-check.d.ts +2 -2
- package/dist/consolidation-undo.d.ts +2 -2
- package/dist/contradiction/index.d.ts +2 -2
- package/dist/conversation-index/backend.d.ts +1 -1
- package/dist/conversation-index/chunker.d.ts +1 -1
- package/dist/conversation-index/faiss-adapter.d.ts +1 -1
- package/dist/conversation-index/indexer.d.ts +1 -1
- package/dist/conversation-index/search.d.ts +1 -1
- package/dist/day-summary.d.ts +1 -1
- package/dist/delinearize.d.ts +1 -1
- package/dist/direct-answer-wiring.d.ts +1 -1
- package/dist/direct-answer.d.ts +1 -1
- package/dist/embedding-fallback.d.ts +1 -1
- package/dist/enrichment/index.d.ts +1 -1
- package/dist/entity-retrieval.d.ts +2 -2
- package/dist/entity-retrieval.js +3 -3
- package/dist/entity-schema.d.ts +1 -1
- package/dist/explicit-capture.d.ts +6 -6
- package/dist/extraction-faithfulness.d.ts +1 -1
- package/dist/extraction-judge-telemetry.d.ts +1 -1
- package/dist/extraction-judge-training.d.ts +1 -1
- package/dist/extraction-judge.d.ts +1 -1
- package/dist/extraction.d.ts +1 -1
- package/dist/fallback-llm.d.ts +1 -1
- package/dist/graph-dashboard-diff.d.ts +1 -1
- package/dist/graph-dashboard-key.d.ts +1 -1
- package/dist/graph-dashboard-parser.d.ts +1 -1
- package/dist/graph-edge-reinforcement.d.ts +1 -1
- package/dist/graph-snapshot.d.ts +1 -1
- package/dist/graph.d.ts +1 -1
- package/dist/importance.d.ts +1 -1
- package/dist/importers/index.d.ts +1 -1
- package/dist/in-flight-reads.d.ts +1 -1
- package/dist/index.d.ts +693 -503
- package/dist/index.js +70 -45
- package/dist/intent.d.ts +1 -1
- package/dist/lcm/engine.d.ts +1 -1
- package/dist/lcm/index.d.ts +1 -1
- package/dist/lcm/tools.d.ts +1 -1
- package/dist/lifecycle.d.ts +1 -1
- package/dist/live-connectors-runner.d.ts +1 -1
- package/dist/local-llm.d.ts +1 -1
- package/dist/local-model-endpoint.d.ts +1 -1
- package/dist/maintenance/memory-governance.d.ts +1 -1
- package/dist/maintenance/memory-governance.js +3 -3
- package/dist/maintenance/rebuild-memory-lifecycle-ledger.d.ts +2 -2
- package/dist/maintenance/rebuild-memory-lifecycle-ledger.js +3 -3
- package/dist/maintenance/rebuild-memory-projection.js +4 -4
- package/dist/mcp-memory-inspector-app.d.ts +8 -8
- package/dist/memory-action-policy.d.ts +1 -1
- package/dist/memory-cache.d.ts +1 -1
- package/dist/memory-lifecycle-ledger-utils.d.ts +1 -1
- package/dist/memory-projection-store.d.ts +1 -1
- package/dist/memory-provenance.d.ts +1 -1
- package/dist/memory-worth-outcomes.d.ts +2 -2
- package/dist/models-json.d.ts +1 -1
- package/dist/namespaces/migrate.d.ts +3 -3
- package/dist/namespaces/migrate.js +4 -4
- package/dist/namespaces/principal.d.ts +1 -1
- package/dist/namespaces/search.d.ts +1 -1
- package/dist/namespaces/storage.d.ts +3 -3
- package/dist/namespaces/storage.js +3 -3
- package/dist/native-knowledge.d.ts +1 -1
- package/dist/offline-sync-impression-drain.d.ts +1 -1
- package/dist/operator-toolkit.d.ts +2 -2
- package/dist/operator-toolkit.js +9 -8
- package/dist/orchestration/compression-guideline-coordinator.d.ts +2 -2
- package/dist/orchestration/maintenance.d.ts +10 -7
- package/dist/orchestration/maintenance.js +7 -6
- package/dist/{orchestrator-BHfWtvEJ.d.ts → orchestrator-BHx-DWCa.d.ts} +7 -5
- package/dist/orchestrator.d.ts +6 -6
- package/dist/orchestrator.js +31 -30
- package/dist/patterns-cli.d.ts +1 -1
- package/dist/{pipeline-CBYcnmDC.d.ts → pipeline-DDHSE876.d.ts} +1 -1
- package/dist/policy-runtime.d.ts +1 -1
- package/dist/proactive-contention.d.ts +1 -1
- package/dist/provenance.d.ts +1 -1
- package/dist/{qmd-BIq-jrV4.d.ts → qmd-C4WVh6BM.d.ts} +1 -1
- package/dist/qmd-preflight.d.ts +2 -2
- package/dist/qmd-recall-cache.d.ts +1 -1
- package/dist/qmd.d.ts +2 -2
- package/dist/recall-concurrency-config.d.ts +1 -1
- package/dist/recall-disclosure-escalation.d.ts +1 -1
- package/dist/recall-explain-renderer.d.ts +1 -1
- package/dist/recall-explain-renderer.js +3 -3
- package/dist/recall-memory-map.d.ts +1 -1
- package/dist/recall-planner-llm.d.ts +1 -1
- package/dist/recall-state.d.ts +1 -1
- package/dist/recall-tag-filter.d.ts +1 -1
- package/dist/recall-timings.d.ts +1 -1
- package/dist/recall-xray-cli.d.ts +1 -1
- package/dist/recall-xray-cli.js +4 -4
- package/dist/recall-xray-renderer.d.ts +1 -1
- package/dist/recall-xray-renderer.js +3 -3
- package/dist/recall-xray.d.ts +1 -1
- package/dist/recall-xray.js +2 -2
- package/dist/replay/normalizers/chatgpt.d.ts +1 -1
- package/dist/replay/normalizers/claude.d.ts +1 -1
- package/dist/replay/normalizers/openclaw.d.ts +1 -1
- package/dist/replay/normalizers/shared.d.ts +1 -1
- package/dist/replay/runner.d.ts +1 -1
- package/dist/replay/types.d.ts +1 -1
- package/dist/resolve-auth-token.d.ts +1 -1
- package/dist/resume-bundles.js +3 -2
- package/dist/retrieval-agents.d.ts +2 -2
- package/dist/retrieval-tiers.d.ts +1 -1
- package/dist/routing/engine.d.ts +1 -1
- package/dist/routing/store.d.ts +1 -1
- package/dist/salvage-envelope.d.ts +1 -1
- package/dist/schemas.d.ts +76 -76
- package/dist/{scope-profiles-DDno-uvy.d.ts → scope-profiles-EqMXwzCz.d.ts} +1 -1
- package/dist/search/embed-helper.d.ts +1 -1
- package/dist/search/factory.d.ts +1 -1
- package/dist/search/index.d.ts +1 -1
- package/dist/search/lancedb-backend.d.ts +1 -1
- package/dist/search/meilisearch-backend.d.ts +1 -1
- package/dist/search/noop-backend.d.ts +1 -1
- package/dist/search/orama-backend.d.ts +1 -1
- package/dist/search/port.d.ts +1 -1
- package/dist/search/remote-backend.d.ts +1 -1
- package/dist/{semantic-consolidation-7RIJFDwY.d.ts → semantic-consolidation-CYpvwmQc.d.ts} +1 -1
- package/dist/semantic-consolidation.d.ts +2 -2
- package/dist/semantic-consolidation.js +4 -4
- package/dist/semantic-rule-promotion.js +3 -3
- package/dist/semantic-rule-verifier.d.ts +1 -1
- package/dist/semantic-rule-verifier.js +3 -3
- package/dist/session-observer-bands.d.ts +1 -1
- package/dist/session-observer-state.d.ts +1 -1
- package/dist/shared-context/manager.d.ts +9 -9
- package/dist/signal.d.ts +1 -1
- package/dist/{storage-DkphZiOJ.d.ts → storage-Dxd7zVZH.d.ts} +1 -1
- package/dist/storage.d.ts +2 -2
- package/dist/storage.js +2 -2
- package/dist/summarizer.d.ts +1 -1
- package/dist/summary-snapshot.d.ts +1 -1
- package/dist/temporal-supersession.d.ts +2 -2
- package/dist/temporal-timeline-recall.d.ts +1 -1
- package/dist/temporal-validity.d.ts +1 -1
- package/dist/threading.d.ts +1 -1
- package/dist/tier-migration.d.ts +2 -2
- package/dist/tier-routing.d.ts +1 -1
- package/dist/topics.d.ts +1 -1
- package/dist/transcript.d.ts +1 -1
- package/dist/transfer/types.d.ts +66 -66
- package/dist/trust-score-stage.d.ts +1 -1
- package/dist/trust-score.d.ts +1 -1
- package/dist/{types-2q_MRFcM.d.ts → types-OUCJALBK.d.ts} +98 -1
- package/dist/types.d.ts +1 -1
- package/dist/types.js +1 -1
- package/dist/utility-runtime.d.ts +1 -1
- package/dist/verified-recall.js +3 -3
- package/dist/write-envelope.d.ts +1 -1
- package/package.json +2 -2
- package/src/activity/config.test.ts +113 -0
- package/src/activity/config.ts +111 -0
- package/src/activity/digest.ts +2 -2
- package/src/activity/index.ts +6 -0
- package/src/activity/pipeline.test.ts +379 -0
- package/src/activity/pipeline.ts +223 -0
- package/src/activity/reindex.test.ts +97 -0
- package/src/activity/reindex.ts +40 -0
- package/src/activity/runner.test.ts +452 -0
- package/src/activity/runner.ts +237 -0
- package/src/activity/scheduler.test.ts +303 -0
- package/src/activity/scheduler.ts +147 -0
- package/src/activity/source-client.test.ts +208 -0
- package/src/activity/source-client.ts +159 -0
- package/src/activity/sync-registration.test.ts +134 -0
- package/src/activity/sync-registration.ts +139 -0
- package/src/activity/types.ts +17 -0
- package/src/config.test.ts +19 -0
- package/src/config.ts +3 -0
- package/src/orchestration/maintenance.test.ts +42 -42
- package/src/orchestration/maintenance.ts +37 -5
- package/src/orchestration/orchestrator-helpers.ts +36 -1
- package/src/orchestration/recall-internal.ts +6 -6
- package/src/orchestration/recall-search-pipeline.ts +15 -7
- package/src/orchestrator-background-writes.test.ts +2 -2
- package/src/orchestrator.ts +3 -3
- package/src/types.ts +4 -1
- package/dist/chunk-CXFVZ3PK.js.map +0 -1
- package/dist/chunk-FJ5IQ3SI.js.map +0 -1
- package/dist/chunk-XJT6LNLY.js.map +0 -1
- package/dist/chunk-YXTSUIXC.js.map +0 -1
- /package/dist/{chunk-IRX5UCGN.js.map → chunk-2OH3ZRYH.js.map} +0 -0
- /package/dist/{chunk-42DJZZUB.js.map → chunk-7LXSPIVG.js.map} +0 -0
- /package/dist/{chunk-47WFEKT7.js.map → chunk-7PK4W4RN.js.map} +0 -0
- /package/dist/{chunk-O7ACJAHR.js.map → chunk-7YJLCNPF.js.map} +0 -0
- /package/dist/{chunk-6EJL2O3G.js.map → chunk-AFH7PIFU.js.map} +0 -0
- /package/dist/{chunk-JN4N2TMQ.js.map → chunk-ANGUOE2N.js.map} +0 -0
- /package/dist/{chunk-GQOKO77V.js.map → chunk-AWT7SCWQ.js.map} +0 -0
- /package/dist/{chunk-PXJEL55X.js.map → chunk-BIRPR5N7.js.map} +0 -0
- /package/dist/{chunk-NAY25BGS.js.map → chunk-BTK2PIFL.js.map} +0 -0
- /package/dist/{chunk-SYCVU556.js.map → chunk-C2RMQRPN.js.map} +0 -0
- /package/dist/{chunk-T4EAQPQQ.js.map → chunk-G2WCVFDC.js.map} +0 -0
- /package/dist/{chunk-UJIBXOJA.js.map → chunk-GEDBTSYU.js.map} +0 -0
- /package/dist/{chunk-WIKOHCQZ.js.map → chunk-I6ODVWQB.js.map} +0 -0
- /package/dist/{chunk-H4MDUPER.js.map → chunk-IZHXSLO7.js.map} +0 -0
- /package/dist/{chunk-RRS5CDB6.js.map → chunk-JOCXI2O5.js.map} +0 -0
- /package/dist/{chunk-EEUZQQRJ.js.map → chunk-KGKYTCDO.js.map} +0 -0
- /package/dist/{chunk-JTW4TGPV.js.map → chunk-L3HAUO7K.js.map} +0 -0
- /package/dist/{chunk-P53HGVJS.js.map → chunk-LMVAIZR5.js.map} +0 -0
- /package/dist/{chunk-KLXG4BBO.js.map → chunk-M6TZQDTX.js.map} +0 -0
- /package/dist/{chunk-U6QWUFSM.js.map → chunk-OXT5TQDS.js.map} +0 -0
- /package/dist/{chunk-T4OTDMX4.js.map → chunk-QDTEH6BS.js.map} +0 -0
- /package/dist/{chunk-5MDE5MIC.js.map → chunk-SESF64QR.js.map} +0 -0
- /package/dist/{chunk-4E5H4FPC.js.map → chunk-TUNI4GB3.js.map} +0 -0
- /package/dist/{chunk-5HFIHWOF.js.map → chunk-VLIY55Q5.js.map} +0 -0
- /package/dist/{chunk-BNH2VVWP.js.map → chunk-WIS5JTJR.js.map} +0 -0
- /package/dist/{chunk-OOGGUXAG.js.map → chunk-WQ2I6ODB.js.map} +0 -0
|
@@ -0,0 +1,452 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import { access, mkdtemp, readFile, rm } from "node:fs/promises";
|
|
3
|
+
import os from "node:os";
|
|
4
|
+
import path from "node:path";
|
|
5
|
+
import test from "node:test";
|
|
6
|
+
|
|
7
|
+
import { activityDatabasePath, ActivityStore } from "./store.js";
|
|
8
|
+
import { activityDigestPath } from "./digest.js";
|
|
9
|
+
import { runActivitySyncOnce } from "./runner.js";
|
|
10
|
+
import { activityCursorKey } from "./pipeline.js";
|
|
11
|
+
import type {
|
|
12
|
+
ActivityConfig,
|
|
13
|
+
ActivitySnapshot,
|
|
14
|
+
ActivitySnapshotPage,
|
|
15
|
+
ActivitySourceClient,
|
|
16
|
+
} from "./types.js";
|
|
17
|
+
|
|
18
|
+
const NOW = new Date("2026-07-22T18:00:00.000Z");
|
|
19
|
+
|
|
20
|
+
function snapshot(overrides: Partial<ActivitySnapshot> = {}): ActivitySnapshot {
|
|
21
|
+
return {
|
|
22
|
+
machine: "wire-payload-label",
|
|
23
|
+
capturedAtUtc: "2026-07-22T14:00:00.000Z",
|
|
24
|
+
app: "Browser",
|
|
25
|
+
windowTitle: "Example",
|
|
26
|
+
text: "A synthetic activity snapshot",
|
|
27
|
+
textSource: "ax",
|
|
28
|
+
contentHash: "hash-1",
|
|
29
|
+
...overrides,
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/** A recording fixture daemon: serves scripted pages keyed by inbound cursor. */
|
|
34
|
+
function fixtureClient(
|
|
35
|
+
machineLabel: string,
|
|
36
|
+
pages: Map<string | null, ActivitySnapshotPage>,
|
|
37
|
+
): ActivitySourceClient & { seenCursors: Array<string | null> } {
|
|
38
|
+
const seenCursors: Array<string | null> = [];
|
|
39
|
+
return {
|
|
40
|
+
machineLabel,
|
|
41
|
+
seenCursors,
|
|
42
|
+
async verify() {
|
|
43
|
+
return { ok: true };
|
|
44
|
+
},
|
|
45
|
+
async fetchSnapshots({ cursor, signal }) {
|
|
46
|
+
signal?.throwIfAborted();
|
|
47
|
+
const key = cursor ?? null;
|
|
48
|
+
seenCursors.push(key);
|
|
49
|
+
const page = pages.get(key);
|
|
50
|
+
if (page === undefined) throw new Error(`fixture has no page for cursor ${String(key)}`);
|
|
51
|
+
return page;
|
|
52
|
+
},
|
|
53
|
+
};
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
function enabledConfig(overrides: Partial<ActivityConfig> = {}): ActivityConfig {
|
|
57
|
+
return {
|
|
58
|
+
enabled: true,
|
|
59
|
+
timezone: "UTC",
|
|
60
|
+
syncDays: 1,
|
|
61
|
+
autoSyncIntervalMinutes: 15,
|
|
62
|
+
sources: [{ machineLabel: "workstation-a", baseUrl: "http://127.0.0.1:8760" }],
|
|
63
|
+
...overrides,
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
test("disabled config makes no client, no HTTP call, and no store write", async () => {
|
|
68
|
+
const memoryDir = await mkdtemp(path.join(os.tmpdir(), "remnic-activity-runner-"));
|
|
69
|
+
try {
|
|
70
|
+
let clientBuilt = false;
|
|
71
|
+
const summary = await runActivitySyncOnce({
|
|
72
|
+
config: {
|
|
73
|
+
enabled: false,
|
|
74
|
+
timezone: "UTC",
|
|
75
|
+
syncDays: 1,
|
|
76
|
+
autoSyncIntervalMinutes: 15,
|
|
77
|
+
sources: [{ machineLabel: "workstation-a", baseUrl: "http://127.0.0.1:8760" }],
|
|
78
|
+
},
|
|
79
|
+
memoryDir,
|
|
80
|
+
now: NOW,
|
|
81
|
+
createSourceClient: () => {
|
|
82
|
+
clientBuilt = true;
|
|
83
|
+
throw new Error("disabled runner must never build a source client");
|
|
84
|
+
},
|
|
85
|
+
});
|
|
86
|
+
|
|
87
|
+
assert.equal(summary.enabled, false);
|
|
88
|
+
assert.deepEqual(summary.results, []);
|
|
89
|
+
assert.equal(summary.ranCount, 0);
|
|
90
|
+
assert.equal(clientBuilt, false, "no client instantiated while disabled");
|
|
91
|
+
await assert.rejects(
|
|
92
|
+
access(activityDatabasePath(memoryDir)),
|
|
93
|
+
"disabled runner must not open (create) the activity store",
|
|
94
|
+
);
|
|
95
|
+
} finally {
|
|
96
|
+
await rm(memoryDir, { recursive: true, force: true });
|
|
97
|
+
}
|
|
98
|
+
});
|
|
99
|
+
|
|
100
|
+
test("enabled source syncs from parsed config: persists, renders digest, advances cursor", async () => {
|
|
101
|
+
const memoryDir = await mkdtemp(path.join(os.tmpdir(), "remnic-activity-runner-"));
|
|
102
|
+
const store = ActivityStore.open(memoryDir);
|
|
103
|
+
try {
|
|
104
|
+
const client = fixtureClient(
|
|
105
|
+
"workstation-a",
|
|
106
|
+
new Map([
|
|
107
|
+
[null, { snapshots: [snapshot()], nextCursor: "c1" }],
|
|
108
|
+
["c1", { snapshots: [snapshot({ contentHash: "hash-2" })], nextCursor: null }],
|
|
109
|
+
]),
|
|
110
|
+
);
|
|
111
|
+
|
|
112
|
+
const summary = await runActivitySyncOnce({
|
|
113
|
+
config: enabledConfig(),
|
|
114
|
+
memoryDir,
|
|
115
|
+
store,
|
|
116
|
+
now: NOW,
|
|
117
|
+
createSourceClient: () => client,
|
|
118
|
+
});
|
|
119
|
+
|
|
120
|
+
assert.equal(summary.enabled, true);
|
|
121
|
+
assert.equal(summary.ranCount, 1);
|
|
122
|
+
assert.equal(summary.errorCount, 0);
|
|
123
|
+
assert.equal(summary.totalInserted, 2);
|
|
124
|
+
assert.deepEqual(summary.results[0], {
|
|
125
|
+
machineLabel: "workstation-a",
|
|
126
|
+
ran: true,
|
|
127
|
+
fetched: 2,
|
|
128
|
+
inserted: 2,
|
|
129
|
+
duplicates: 0,
|
|
130
|
+
digestsWritten: 1,
|
|
131
|
+
cursor: "c1",
|
|
132
|
+
});
|
|
133
|
+
assert.equal(store.getCursor(activityCursorKey("workstation-a", "2026-07-22")), "c1");
|
|
134
|
+
assert.match(await readFile(activityDigestPath(memoryDir, "2026-07-22"), "utf8"), /snapshotCount: 2/);
|
|
135
|
+
} finally {
|
|
136
|
+
store.close();
|
|
137
|
+
await rm(memoryDir, { recursive: true, force: true });
|
|
138
|
+
}
|
|
139
|
+
});
|
|
140
|
+
|
|
141
|
+
test("multi-source: one source's fault is isolated and never advances the other", async () => {
|
|
142
|
+
const memoryDir = await mkdtemp(path.join(os.tmpdir(), "remnic-activity-runner-"));
|
|
143
|
+
const store = ActivityStore.open(memoryDir);
|
|
144
|
+
try {
|
|
145
|
+
const healthy = fixtureClient(
|
|
146
|
+
"workstation-a",
|
|
147
|
+
new Map([[null, { snapshots: [snapshot()], nextCursor: null }]]),
|
|
148
|
+
);
|
|
149
|
+
const faulty: ActivitySourceClient = {
|
|
150
|
+
machineLabel: "workstation-b",
|
|
151
|
+
async verify() {
|
|
152
|
+
return { ok: true };
|
|
153
|
+
},
|
|
154
|
+
async fetchSnapshots() {
|
|
155
|
+
// A network-shaped error whose message embeds a host: it must be
|
|
156
|
+
// sanitized to a name+code, not echoed verbatim, on ActivitySourceRunItem.
|
|
157
|
+
throw Object.assign(new Error("daemon connection refused at 10.0.0.5:8761"), { code: "ECONNREFUSED" });
|
|
158
|
+
},
|
|
159
|
+
};
|
|
160
|
+
|
|
161
|
+
const summary = await runActivitySyncOnce({
|
|
162
|
+
config: enabledConfig({
|
|
163
|
+
sources: [
|
|
164
|
+
{ machineLabel: "workstation-a", baseUrl: "http://127.0.0.1:8760" },
|
|
165
|
+
{ machineLabel: "workstation-b", baseUrl: "http://127.0.0.1:8761" },
|
|
166
|
+
],
|
|
167
|
+
}),
|
|
168
|
+
memoryDir,
|
|
169
|
+
store,
|
|
170
|
+
now: NOW,
|
|
171
|
+
createSourceClient: (source) => (source.machineLabel === "workstation-a" ? healthy : faulty),
|
|
172
|
+
});
|
|
173
|
+
|
|
174
|
+
assert.equal(summary.ranCount, 1);
|
|
175
|
+
assert.equal(summary.errorCount, 1);
|
|
176
|
+
const byLabel = new Map(summary.results.map((item) => [item.machineLabel, item]));
|
|
177
|
+
assert.equal(byLabel.get("workstation-a")?.ran, true);
|
|
178
|
+
assert.equal(byLabel.get("workstation-a")?.inserted, 1);
|
|
179
|
+
assert.equal(byLabel.get("workstation-a")?.error, undefined);
|
|
180
|
+
assert.equal(byLabel.get("workstation-b")?.ran, false);
|
|
181
|
+
assert.match(byLabel.get("workstation-b")?.error ?? "", /ECONNREFUSED/);
|
|
182
|
+
assert.ok(!byLabel.get("workstation-b")?.error?.includes("10.0.0.5"), "raw host is sanitized out of the error");
|
|
183
|
+
// The healthy source still committed; the faulty one never wrote a cursor.
|
|
184
|
+
assert.equal(store.getCursor(activityCursorKey("workstation-b", "2026-07-22")), null);
|
|
185
|
+
} finally {
|
|
186
|
+
store.close();
|
|
187
|
+
await rm(memoryDir, { recursive: true, force: true });
|
|
188
|
+
}
|
|
189
|
+
});
|
|
190
|
+
|
|
191
|
+
test("error stays distinguishable from an empty page", async () => {
|
|
192
|
+
const memoryDir = await mkdtemp(path.join(os.tmpdir(), "remnic-activity-runner-"));
|
|
193
|
+
const store = ActivityStore.open(memoryDir);
|
|
194
|
+
try {
|
|
195
|
+
const empty = fixtureClient(
|
|
196
|
+
"empty-src",
|
|
197
|
+
new Map([[null, { snapshots: [], nextCursor: null }]]),
|
|
198
|
+
);
|
|
199
|
+
const faulty: ActivitySourceClient = {
|
|
200
|
+
machineLabel: "fault-src",
|
|
201
|
+
async verify() {
|
|
202
|
+
return { ok: true };
|
|
203
|
+
},
|
|
204
|
+
async fetchSnapshots() {
|
|
205
|
+
throw new Error("HTTP 503");
|
|
206
|
+
},
|
|
207
|
+
};
|
|
208
|
+
|
|
209
|
+
const summary = await runActivitySyncOnce({
|
|
210
|
+
config: enabledConfig({
|
|
211
|
+
sources: [
|
|
212
|
+
{ machineLabel: "empty-src", baseUrl: "http://127.0.0.1:8760" },
|
|
213
|
+
{ machineLabel: "fault-src", baseUrl: "http://127.0.0.1:8761" },
|
|
214
|
+
],
|
|
215
|
+
}),
|
|
216
|
+
memoryDir,
|
|
217
|
+
store,
|
|
218
|
+
now: NOW,
|
|
219
|
+
createSourceClient: (source) => (source.machineLabel === "empty-src" ? empty : faulty),
|
|
220
|
+
});
|
|
221
|
+
|
|
222
|
+
const byLabel = new Map(summary.results.map((item) => [item.machineLabel, item]));
|
|
223
|
+
// Empty page: a real, successful sync that fetched and inserted nothing.
|
|
224
|
+
assert.equal(byLabel.get("empty-src")?.ran, true);
|
|
225
|
+
assert.equal(byLabel.get("empty-src")?.inserted, 0);
|
|
226
|
+
assert.equal(byLabel.get("empty-src")?.error, undefined);
|
|
227
|
+
// Fault: surfaced as an error, not silently conflated with an empty result.
|
|
228
|
+
assert.equal(byLabel.get("fault-src")?.ran, false);
|
|
229
|
+
assert.notEqual(byLabel.get("fault-src")?.error, undefined);
|
|
230
|
+
} finally {
|
|
231
|
+
store.close();
|
|
232
|
+
await rm(memoryDir, { recursive: true, force: true });
|
|
233
|
+
}
|
|
234
|
+
});
|
|
235
|
+
|
|
236
|
+
test("stop: an aborted run leaves the cursor unadvanced", async () => {
|
|
237
|
+
const memoryDir = await mkdtemp(path.join(os.tmpdir(), "remnic-activity-runner-"));
|
|
238
|
+
const store = ActivityStore.open(memoryDir);
|
|
239
|
+
try {
|
|
240
|
+
const client = fixtureClient(
|
|
241
|
+
"workstation-a",
|
|
242
|
+
new Map([[null, { snapshots: [snapshot()], nextCursor: null }]]),
|
|
243
|
+
);
|
|
244
|
+
|
|
245
|
+
const summary = await runActivitySyncOnce({
|
|
246
|
+
config: enabledConfig(),
|
|
247
|
+
memoryDir,
|
|
248
|
+
store,
|
|
249
|
+
now: NOW,
|
|
250
|
+
signal: AbortSignal.abort(),
|
|
251
|
+
createSourceClient: () => client,
|
|
252
|
+
});
|
|
253
|
+
|
|
254
|
+
assert.equal(summary.errorCount, 1);
|
|
255
|
+
assert.equal(summary.ranCount, 0);
|
|
256
|
+
assert.equal(client.seenCursors.length, 0, "aborted before any fetch reached the daemon");
|
|
257
|
+
assert.equal(store.getCursor(activityCursorKey("workstation-a", "2026-07-22")), null, "no cursor persisted on abort");
|
|
258
|
+
} finally {
|
|
259
|
+
store.close();
|
|
260
|
+
await rm(memoryDir, { recursive: true, force: true });
|
|
261
|
+
}
|
|
262
|
+
});
|
|
263
|
+
|
|
264
|
+
test("abort halts the whole run: later sources never build a client or sync", async () => {
|
|
265
|
+
const memoryDir = await mkdtemp(path.join(os.tmpdir(), "remnic-activity-runner-"));
|
|
266
|
+
const store = ActivityStore.open(memoryDir);
|
|
267
|
+
try {
|
|
268
|
+
const built: string[] = [];
|
|
269
|
+
const summary = await runActivitySyncOnce({
|
|
270
|
+
config: enabledConfig({
|
|
271
|
+
sources: [
|
|
272
|
+
{ machineLabel: "workstation-a", baseUrl: "http://127.0.0.1:8760" },
|
|
273
|
+
{ machineLabel: "workstation-b", baseUrl: "http://127.0.0.1:8761" },
|
|
274
|
+
],
|
|
275
|
+
}),
|
|
276
|
+
memoryDir,
|
|
277
|
+
store,
|
|
278
|
+
now: NOW,
|
|
279
|
+
signal: AbortSignal.abort(),
|
|
280
|
+
createSourceClient: (source) => {
|
|
281
|
+
built.push(source.machineLabel);
|
|
282
|
+
return fixtureClient(source.machineLabel, new Map([[null, { snapshots: [snapshot()], nextCursor: null }]]));
|
|
283
|
+
},
|
|
284
|
+
});
|
|
285
|
+
|
|
286
|
+
// The first source's fast-failing (aborted) attempt is recorded; the run
|
|
287
|
+
// then halts — the second source is never even instantiated.
|
|
288
|
+
assert.deepEqual(built, ["workstation-a"], "abort stops the run before later sources build a client");
|
|
289
|
+
assert.equal(summary.results.length, 1);
|
|
290
|
+
assert.equal(summary.ranCount, 0);
|
|
291
|
+
assert.equal(summary.errorCount, 1);
|
|
292
|
+
assert.equal(store.getCursor(activityCursorKey("workstation-b", "2026-07-22")), null);
|
|
293
|
+
} finally {
|
|
294
|
+
store.close();
|
|
295
|
+
await rm(memoryDir, { recursive: true, force: true });
|
|
296
|
+
}
|
|
297
|
+
});
|
|
298
|
+
|
|
299
|
+
test("restart: a fresh run resumes from the persisted cursor", async () => {
|
|
300
|
+
const memoryDir = await mkdtemp(path.join(os.tmpdir(), "remnic-activity-runner-"));
|
|
301
|
+
const store = ActivityStore.open(memoryDir);
|
|
302
|
+
try {
|
|
303
|
+
const first = fixtureClient(
|
|
304
|
+
"workstation-a",
|
|
305
|
+
new Map([
|
|
306
|
+
[null, { snapshots: [snapshot()], nextCursor: "c1" }],
|
|
307
|
+
["c1", { snapshots: [snapshot({ contentHash: "hash-2" })], nextCursor: null }],
|
|
308
|
+
]),
|
|
309
|
+
);
|
|
310
|
+
await runActivitySyncOnce({
|
|
311
|
+
config: enabledConfig(),
|
|
312
|
+
memoryDir,
|
|
313
|
+
store,
|
|
314
|
+
now: NOW,
|
|
315
|
+
createSourceClient: () => first,
|
|
316
|
+
});
|
|
317
|
+
assert.equal(store.getCursor(activityCursorKey("workstation-a", "2026-07-22")), "c1");
|
|
318
|
+
|
|
319
|
+
// Simulate a process restart: a brand-new client that only knows how to
|
|
320
|
+
// continue from the persisted cursor. If the runner ignored it, this
|
|
321
|
+
// fixture would throw ("no page for cursor null").
|
|
322
|
+
const second = fixtureClient(
|
|
323
|
+
"workstation-a",
|
|
324
|
+
new Map([["c1", { snapshots: [snapshot({ contentHash: "hash-3" })], nextCursor: null }]]),
|
|
325
|
+
);
|
|
326
|
+
const summary = await runActivitySyncOnce({
|
|
327
|
+
config: enabledConfig(),
|
|
328
|
+
memoryDir,
|
|
329
|
+
store,
|
|
330
|
+
now: NOW,
|
|
331
|
+
createSourceClient: () => second,
|
|
332
|
+
});
|
|
333
|
+
|
|
334
|
+
assert.deepEqual(second.seenCursors, ["c1"], "restart resumed from the stored cursor");
|
|
335
|
+
assert.equal(summary.results[0]?.inserted, 1);
|
|
336
|
+
assert.equal(store.getCursor(activityCursorKey("workstation-a", "2026-07-22")), "c1");
|
|
337
|
+
} finally {
|
|
338
|
+
store.close();
|
|
339
|
+
await rm(memoryDir, { recursive: true, force: true });
|
|
340
|
+
}
|
|
341
|
+
});
|
|
342
|
+
|
|
343
|
+
test("reindexSearch is forwarded per source and a failure never fails the run", async () => {
|
|
344
|
+
const memoryDir = await mkdtemp(path.join(os.tmpdir(), "remnic-activity-runner-"));
|
|
345
|
+
const store = ActivityStore.open(memoryDir);
|
|
346
|
+
try {
|
|
347
|
+
const client = fixtureClient(
|
|
348
|
+
"workstation-a",
|
|
349
|
+
new Map([[null, { snapshots: [snapshot()], nextCursor: null }]]),
|
|
350
|
+
);
|
|
351
|
+
let reindexCalls = 0;
|
|
352
|
+
const summary = await runActivitySyncOnce({
|
|
353
|
+
config: enabledConfig(),
|
|
354
|
+
memoryDir,
|
|
355
|
+
store,
|
|
356
|
+
now: NOW,
|
|
357
|
+
createSourceClient: () => client,
|
|
358
|
+
reindexSearch: async () => {
|
|
359
|
+
reindexCalls += 1;
|
|
360
|
+
// Realistic forced-strict failure surfaced by updateCollectionStrict.
|
|
361
|
+
throw new Error("QMD update skipped by per-collection failure backoff");
|
|
362
|
+
},
|
|
363
|
+
});
|
|
364
|
+
|
|
365
|
+
assert.equal(reindexCalls, 1, "the runner forwards reindexSearch to the source sync");
|
|
366
|
+
// Durable write still committed despite the reindex failure:
|
|
367
|
+
assert.equal(summary.ranCount, 1, "a reindex failure does not fail the source");
|
|
368
|
+
assert.equal(summary.errorCount, 0, "a reindex failure is not a sync error");
|
|
369
|
+
assert.equal(summary.results[0]?.ran, true);
|
|
370
|
+
assert.equal(summary.results[0]?.inserted, 1);
|
|
371
|
+
// ...but the failed refresh is surfaced as a signal, not a silent clean run:
|
|
372
|
+
assert.equal(summary.reindexErrorCount, 1, "the failed refresh is counted");
|
|
373
|
+
assert.match(summary.results[0]?.reindexError ?? "", /failure backoff/);
|
|
374
|
+
} finally {
|
|
375
|
+
store.close();
|
|
376
|
+
await rm(memoryDir, { recursive: true, force: true });
|
|
377
|
+
}
|
|
378
|
+
});
|
|
379
|
+
|
|
380
|
+
test("partial multi-day sync reports ran=true for days that synced before a later failure", async () => {
|
|
381
|
+
const memoryDir = await mkdtemp(path.join(os.tmpdir(), "remnic-activity-runner-"));
|
|
382
|
+
const store = ActivityStore.open(memoryDir);
|
|
383
|
+
try {
|
|
384
|
+
let calls = 0;
|
|
385
|
+
const client: ActivitySourceClient = {
|
|
386
|
+
machineLabel: "workstation-a",
|
|
387
|
+
async verify() {
|
|
388
|
+
return { ok: true };
|
|
389
|
+
},
|
|
390
|
+
async fetchSnapshots() {
|
|
391
|
+
calls += 1;
|
|
392
|
+
// First local day (oldest) syncs; the second day's daemon call fails.
|
|
393
|
+
if (calls === 1) return { snapshots: [snapshot({ capturedAtUtc: "2026-07-21T14:00:00.000Z" })], nextCursor: null };
|
|
394
|
+
throw new Error("activity source HTTP 503");
|
|
395
|
+
},
|
|
396
|
+
};
|
|
397
|
+
|
|
398
|
+
const summary = await runActivitySyncOnce({
|
|
399
|
+
config: enabledConfig({ syncDays: 2 }),
|
|
400
|
+
memoryDir,
|
|
401
|
+
store,
|
|
402
|
+
now: NOW,
|
|
403
|
+
createSourceClient: () => client,
|
|
404
|
+
});
|
|
405
|
+
|
|
406
|
+
const item = summary.results[0];
|
|
407
|
+
assert.equal(item?.ran, true, "the earlier day synced durably, so ran is true despite the later failure");
|
|
408
|
+
assert.equal(item?.inserted, 1);
|
|
409
|
+
assert.match(item?.error ?? "", /HTTP 503/);
|
|
410
|
+
assert.equal(summary.ranCount, 1, "a partially-synced source still counts as ran");
|
|
411
|
+
} finally {
|
|
412
|
+
store.close();
|
|
413
|
+
await rm(memoryDir, { recursive: true, force: true });
|
|
414
|
+
}
|
|
415
|
+
});
|
|
416
|
+
|
|
417
|
+
test("an older-day failure does not starve newer dates in a multi-day window", async () => {
|
|
418
|
+
const memoryDir = await mkdtemp(path.join(os.tmpdir(), "remnic-activity-runner-"));
|
|
419
|
+
const store = ActivityStore.open(memoryDir);
|
|
420
|
+
try {
|
|
421
|
+
let calls = 0;
|
|
422
|
+
const client: ActivitySourceClient = {
|
|
423
|
+
machineLabel: "workstation-a",
|
|
424
|
+
async verify() {
|
|
425
|
+
return { ok: true };
|
|
426
|
+
},
|
|
427
|
+
async fetchSnapshots() {
|
|
428
|
+
calls += 1;
|
|
429
|
+
// Dates are oldest-first: the first call is the OLDER day and fails;
|
|
430
|
+
// the newer day (today) must still sync.
|
|
431
|
+
if (calls === 1) throw new Error("activity source HTTP 500");
|
|
432
|
+
return { snapshots: [snapshot()], nextCursor: null };
|
|
433
|
+
},
|
|
434
|
+
};
|
|
435
|
+
|
|
436
|
+
const summary = await runActivitySyncOnce({
|
|
437
|
+
config: enabledConfig({ syncDays: 2 }),
|
|
438
|
+
memoryDir,
|
|
439
|
+
store,
|
|
440
|
+
now: NOW,
|
|
441
|
+
createSourceClient: () => client,
|
|
442
|
+
});
|
|
443
|
+
|
|
444
|
+
const item = summary.results[0];
|
|
445
|
+
assert.equal(item?.ran, true, "the newer day synced despite the older day's failure");
|
|
446
|
+
assert.equal(item?.inserted, 1, "the newer day's snapshot was persisted");
|
|
447
|
+
assert.match(item?.error ?? "", /HTTP 500/, "the failed older day is still reported");
|
|
448
|
+
} finally {
|
|
449
|
+
store.close();
|
|
450
|
+
await rm(memoryDir, { recursive: true, force: true });
|
|
451
|
+
}
|
|
452
|
+
});
|
|
@@ -0,0 +1,237 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Host-agnostic activity sync runner (issue #1900).
|
|
3
|
+
*
|
|
4
|
+
* Bridges parsed `config.activity` to the durable sync pipeline: the parser
|
|
5
|
+
* (config.ts) validates settings but never contacts a daemon, and
|
|
6
|
+
* `syncActivitySource` (pipeline.ts) syncs one source for one day. This runner
|
|
7
|
+
* is the missing middle — it instantiates one `ActivityHttpSourceClient` per
|
|
8
|
+
* configured source from the parsed config and drives a sync across the
|
|
9
|
+
* `syncDays` local-day window, but ONLY when the feature is enabled.
|
|
10
|
+
*
|
|
11
|
+
* The feature is master default-off: source config alone never starts a sync.
|
|
12
|
+
* When `config.enabled` is false the runner opens no store, builds no client,
|
|
13
|
+
* and makes no HTTP call or write — it returns a disabled summary. This mirrors
|
|
14
|
+
* `runLiveConnectorsOnce`'s disabled-vs-result reporting so a host scheduler
|
|
15
|
+
* (CLI, cron) can trigger it through the runtime with no OpenClaw imports.
|
|
16
|
+
*
|
|
17
|
+
* Cursor semantics are delegated wholesale to `syncActivitySource`: the
|
|
18
|
+
* per-machine cursor advances only after every page and the digest are durable,
|
|
19
|
+
* so a runtime failure (or an aborted stop) leaves the cursor unadvanced and a
|
|
20
|
+
* restart resumes from the persisted position. A per-source failure is captured
|
|
21
|
+
* as an error item and never aborts the other sources or the host flow.
|
|
22
|
+
*/
|
|
23
|
+
|
|
24
|
+
import { ActivityStore } from "./store.js";
|
|
25
|
+
import { syncActivitySource } from "./pipeline.js";
|
|
26
|
+
import { ActivityHttpSourceClient } from "./source-client.js";
|
|
27
|
+
import { displayErrorDetail } from "../runtime/better-sqlite.js";
|
|
28
|
+
import type { ActivityConfig, ActivitySourceClient, ActivitySourceConfig } from "./types.js";
|
|
29
|
+
|
|
30
|
+
type ActivityNow = Date | (() => Date);
|
|
31
|
+
|
|
32
|
+
/** One configured source's aggregate result across the synced day window. */
|
|
33
|
+
export interface ActivitySourceRunItem {
|
|
34
|
+
machineLabel: string;
|
|
35
|
+
/** True when at least one day synced durably (no error). */
|
|
36
|
+
ran: boolean;
|
|
37
|
+
fetched: number;
|
|
38
|
+
inserted: number;
|
|
39
|
+
duplicates: number;
|
|
40
|
+
digestsWritten: number;
|
|
41
|
+
/** Persisted cursor after the run; unadvanced from prior on failure. */
|
|
42
|
+
cursor: string | null;
|
|
43
|
+
/** Set when the source failed; distinguishes a fault from an empty page. */
|
|
44
|
+
error?: string;
|
|
45
|
+
/**
|
|
46
|
+
* Set when a durable day synced but its post-digest search reindex failed.
|
|
47
|
+
* The rows/digest/cursor are committed; the index is stale until the next
|
|
48
|
+
* refresh, so this is a signal (not a sync failure) for the scheduler.
|
|
49
|
+
*/
|
|
50
|
+
reindexError?: string;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export interface ActivitySyncRunSummary {
|
|
54
|
+
ranAt: string;
|
|
55
|
+
/** False means the feature is off: no client built, no HTTP, no write. */
|
|
56
|
+
enabled: boolean;
|
|
57
|
+
ranCount: number;
|
|
58
|
+
errorCount: number;
|
|
59
|
+
/** Sources whose durable sync succeeded but whose search reindex failed. */
|
|
60
|
+
reindexErrorCount: number;
|
|
61
|
+
totalInserted: number;
|
|
62
|
+
results: ActivitySourceRunItem[];
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
export interface ActivitySyncRunOptions {
|
|
66
|
+
config: ActivityConfig;
|
|
67
|
+
memoryDir: string;
|
|
68
|
+
/** Injectable store; opened from memoryDir when omitted (and closed here). */
|
|
69
|
+
store?: ActivityStore;
|
|
70
|
+
/** Clock injection for deterministic day-window resolution in tests. */
|
|
71
|
+
now?: ActivityNow;
|
|
72
|
+
/** Abort a run in flight (stop); an aborted source leaves its cursor intact. */
|
|
73
|
+
signal?: AbortSignal;
|
|
74
|
+
/** Injectable client factory; builds an ActivityHttpSourceClient by default. */
|
|
75
|
+
createSourceClient?: (source: ActivitySourceConfig) => ActivitySourceClient;
|
|
76
|
+
/**
|
|
77
|
+
* Search-index refresh forwarded to each source sync as `afterWrites`, run
|
|
78
|
+
* after a digest is (re)written so it becomes discoverable (rule 31).
|
|
79
|
+
* Host wires the core index seam (SearchBackend.update); best-effort.
|
|
80
|
+
*/
|
|
81
|
+
reindexSearch?: (signal?: AbortSignal) => Promise<void>;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/** Format a Date as YYYY-MM-DD in an IANA timezone (local calendar day). */
|
|
85
|
+
function localDateInTimezone(instant: Date, timezone: string): string {
|
|
86
|
+
try {
|
|
87
|
+
const parts = new Intl.DateTimeFormat("en-CA", {
|
|
88
|
+
timeZone: timezone,
|
|
89
|
+
year: "numeric",
|
|
90
|
+
month: "2-digit",
|
|
91
|
+
day: "2-digit",
|
|
92
|
+
}).formatToParts(instant);
|
|
93
|
+
const get = (type: string): string => parts.find((part) => part.type === type)?.value ?? "";
|
|
94
|
+
return `${get("year")}-${get("month")}-${get("day")}`;
|
|
95
|
+
} catch {
|
|
96
|
+
return instant.toISOString().slice(0, 10);
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/** Previous calendar date via pure date arithmetic (no DST wall-clock drift). */
|
|
101
|
+
function previousIsoDate(date: string): string {
|
|
102
|
+
const parsed = new Date(`${date}T00:00:00Z`);
|
|
103
|
+
parsed.setUTCDate(parsed.getUTCDate() - 1);
|
|
104
|
+
return parsed.toISOString().slice(0, 10);
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/**
|
|
108
|
+
* The local-day window to sync, oldest first: today back `syncDays` calendar
|
|
109
|
+
* days in the configured timezone. Walking by calendar days (not fixed 24h
|
|
110
|
+
* intervals) keeps the window correct across DST transitions.
|
|
111
|
+
*/
|
|
112
|
+
export function resolveActivitySyncDates(syncDays: number, timezone: string, now: Date): string[] {
|
|
113
|
+
const dates: string[] = [];
|
|
114
|
+
let cursor = localDateInTimezone(now, timezone);
|
|
115
|
+
for (let count = 0; count < syncDays; count++) {
|
|
116
|
+
dates.unshift(cursor);
|
|
117
|
+
cursor = previousIsoDate(cursor);
|
|
118
|
+
}
|
|
119
|
+
return dates;
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/**
|
|
123
|
+
* Operator-facing detail for a per-source/per-day sync fault. Our own
|
|
124
|
+
* controlled messages (activity.../Invalid activity...) are informative and
|
|
125
|
+
* safe to surface; opaque network/runtime errors can embed hostnames or paths,
|
|
126
|
+
* so they are reduced to a sanitized name+code (matches ActivityHttpSourceClient.verify).
|
|
127
|
+
*/
|
|
128
|
+
function sanitizedSyncError(error: unknown): string {
|
|
129
|
+
return error instanceof Error && /^(activity|Invalid activity)/.test(error.message)
|
|
130
|
+
? error.message
|
|
131
|
+
: displayErrorDetail(error) || "activity sync failed";
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
/**
|
|
135
|
+
* Run one activity sync pass across every configured source. Disabled config
|
|
136
|
+
* short-circuits before any store/client/HTTP work; an enabled config syncs
|
|
137
|
+
* each source across the `syncDays` window and isolates per-source faults.
|
|
138
|
+
*/
|
|
139
|
+
export async function runActivitySyncOnce(options: ActivitySyncRunOptions): Promise<ActivitySyncRunSummary> {
|
|
140
|
+
const ranAt = typeof options.now === "function" ? options.now() : options.now ?? new Date();
|
|
141
|
+
|
|
142
|
+
if (!options.config.enabled) {
|
|
143
|
+
return {
|
|
144
|
+
ranAt: ranAt.toISOString(),
|
|
145
|
+
enabled: false,
|
|
146
|
+
ranCount: 0,
|
|
147
|
+
errorCount: 0,
|
|
148
|
+
reindexErrorCount: 0,
|
|
149
|
+
totalInserted: 0,
|
|
150
|
+
results: [],
|
|
151
|
+
};
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
const dates = resolveActivitySyncDates(options.config.syncDays, options.config.timezone, ranAt);
|
|
155
|
+
const createSourceClient =
|
|
156
|
+
options.createSourceClient ??
|
|
157
|
+
((source: ActivitySourceConfig): ActivitySourceClient =>
|
|
158
|
+
new ActivityHttpSourceClient({
|
|
159
|
+
machineLabel: source.machineLabel,
|
|
160
|
+
baseUrl: source.baseUrl,
|
|
161
|
+
...(source.token === undefined ? {} : { token: source.token }),
|
|
162
|
+
}));
|
|
163
|
+
const ownStore = options.store === undefined;
|
|
164
|
+
const store = options.store ?? ActivityStore.open(options.memoryDir);
|
|
165
|
+
const results: ActivitySourceRunItem[] = [];
|
|
166
|
+
|
|
167
|
+
try {
|
|
168
|
+
for (const sourceConfig of options.config.sources) {
|
|
169
|
+
const item: ActivitySourceRunItem = {
|
|
170
|
+
machineLabel: sourceConfig.machineLabel,
|
|
171
|
+
ran: false,
|
|
172
|
+
fetched: 0,
|
|
173
|
+
inserted: 0,
|
|
174
|
+
duplicates: 0,
|
|
175
|
+
digestsWritten: 0,
|
|
176
|
+
// Cursors are now per (machine, date); report the last synced day's
|
|
177
|
+
// cursor. Null until a day advances it.
|
|
178
|
+
cursor: null,
|
|
179
|
+
};
|
|
180
|
+
try {
|
|
181
|
+
const client = createSourceClient(sourceConfig);
|
|
182
|
+
for (const date of dates) {
|
|
183
|
+
try {
|
|
184
|
+
const result = await syncActivitySource(client, {
|
|
185
|
+
date,
|
|
186
|
+
timezone: options.config.timezone,
|
|
187
|
+
memoryDir: options.memoryDir,
|
|
188
|
+
store,
|
|
189
|
+
signal: options.signal,
|
|
190
|
+
...(options.reindexSearch === undefined ? {} : { afterWrites: options.reindexSearch }),
|
|
191
|
+
});
|
|
192
|
+
item.fetched += result.fetched;
|
|
193
|
+
item.inserted += result.inserted;
|
|
194
|
+
item.duplicates += result.duplicates;
|
|
195
|
+
if (result.digestWritten) item.digestsWritten += 1;
|
|
196
|
+
item.cursor = result.cursor;
|
|
197
|
+
// A day that synced durably counts as ran, even if a later day in
|
|
198
|
+
// the window fails (ran = at least one durable day).
|
|
199
|
+
item.ran = true;
|
|
200
|
+
// The day's rows/digest/cursor are durable; a failed post-digest
|
|
201
|
+
// reindex is a separate, already-sanitized signal (not a sync
|
|
202
|
+
// failure) so the scheduler can observe a stale index.
|
|
203
|
+
if (result.reindexError !== undefined) item.reindexError = result.reindexError;
|
|
204
|
+
} catch (dateError) {
|
|
205
|
+
// One bad day (a malformed historical snapshot, a per-day daemon
|
|
206
|
+
// error) must not starve the remaining dates — record it and move
|
|
207
|
+
// on. Dates are processed oldest-first, so continuing keeps today's
|
|
208
|
+
// sync alive when an old backfill day is broken. An abort, though,
|
|
209
|
+
// halts the whole run.
|
|
210
|
+
item.error = sanitizedSyncError(dateError);
|
|
211
|
+
if (options.signal?.aborted) break;
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
} catch (error) {
|
|
215
|
+
// Client construction failed: no date could run for this source.
|
|
216
|
+
item.error = sanitizedSyncError(error);
|
|
217
|
+
}
|
|
218
|
+
results.push(item);
|
|
219
|
+
// An abort halts the WHOLE run: the in-flight source above is recorded
|
|
220
|
+
// (its fast-failing attempt reports an error), but no later source builds
|
|
221
|
+
// a client or contacts a daemon — teardown settles promptly.
|
|
222
|
+
if (options.signal?.aborted) break;
|
|
223
|
+
}
|
|
224
|
+
} finally {
|
|
225
|
+
if (ownStore) store.close();
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
return {
|
|
229
|
+
ranAt: ranAt.toISOString(),
|
|
230
|
+
enabled: true,
|
|
231
|
+
ranCount: results.filter((item) => item.ran).length,
|
|
232
|
+
errorCount: results.filter((item) => item.error !== undefined).length,
|
|
233
|
+
reindexErrorCount: results.filter((item) => item.reindexError !== undefined).length,
|
|
234
|
+
totalInserted: results.reduce((sum, item) => sum + item.inserted, 0),
|
|
235
|
+
results,
|
|
236
|
+
};
|
|
237
|
+
}
|