@cyanheads/mcp-ts-core 0.9.16 → 0.9.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +3 -2
- package/CLAUDE.md +3 -2
- package/README.md +1 -1
- package/changelog/0.9.x/0.9.17.md +23 -0
- package/changelog/0.9.x/0.9.18.md +12 -0
- package/dist/mcp-server/tools/utils/toolDefinition.d.ts.map +1 -1
- package/dist/mcp-server/tools/utils/toolDefinition.js.map +1 -1
- package/dist/services/mirror/core/defineMirror.d.ts +62 -0
- package/dist/services/mirror/core/defineMirror.d.ts.map +1 -0
- package/dist/services/mirror/core/defineMirror.js +99 -0
- package/dist/services/mirror/core/defineMirror.js.map +1 -0
- package/dist/services/mirror/core/runner.d.ts +29 -0
- package/dist/services/mirror/core/runner.d.ts.map +1 -0
- package/dist/services/mirror/core/runner.js +95 -0
- package/dist/services/mirror/core/runner.js.map +1 -0
- package/dist/services/mirror/index.d.ts +18 -0
- package/dist/services/mirror/index.d.ts.map +1 -0
- package/dist/services/mirror/index.js +17 -0
- package/dist/services/mirror/index.js.map +1 -0
- package/dist/services/mirror/sqlite/handle.d.ts +48 -0
- package/dist/services/mirror/sqlite/handle.d.ts.map +1 -0
- package/dist/services/mirror/sqlite/handle.js +105 -0
- package/dist/services/mirror/sqlite/handle.js.map +1 -0
- package/dist/services/mirror/sqlite/schema.d.ts +38 -0
- package/dist/services/mirror/sqlite/schema.d.ts.map +1 -0
- package/dist/services/mirror/sqlite/schema.js +124 -0
- package/dist/services/mirror/sqlite/schema.js.map +1 -0
- package/dist/services/mirror/sqlite/sqliteMirrorStore.d.ts +27 -0
- package/dist/services/mirror/sqlite/sqliteMirrorStore.d.ts.map +1 -0
- package/dist/services/mirror/sqlite/sqliteMirrorStore.js +286 -0
- package/dist/services/mirror/sqlite/sqliteMirrorStore.js.map +1 -0
- package/dist/services/mirror/types.d.ts +211 -0
- package/dist/services/mirror/types.d.ts.map +1 -0
- package/dist/services/mirror/types.js +9 -0
- package/dist/services/mirror/types.js.map +1 -0
- package/dist/utils/internal/error-handler/types.d.ts.map +1 -1
- package/dist/utils/internal/requestContext.d.ts.map +1 -1
- package/dist/utils/internal/requestContext.js.map +1 -1
- package/dist/utils/pagination/pagination.d.ts.map +1 -1
- package/dist/utils/pagination/pagination.js.map +1 -1
- package/package.json +12 -2
- package/scripts/check-skill-versions.ts +137 -0
- package/scripts/devcheck.ts +39 -0
- package/skills/api-mirror/SKILL.md +103 -0
- package/skills/design-mcp-server/SKILL.md +2 -1
- package/dist/logs/combined.log +0 -4
- package/dist/logs/error.log +0 -2
- package/dist/logs/interactions.log +0 -0
package/AGENTS.md
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# Developer Protocol
|
|
2
2
|
|
|
3
3
|
**Package:** `@cyanheads/mcp-ts-core`
|
|
4
|
-
**Version:** 0.9.
|
|
4
|
+
**Version:** 0.9.18
|
|
5
5
|
**Engines:** Bun ≥1.3.0, Node ≥24.0.0
|
|
6
6
|
**MCP SDK:** `@modelcontextprotocol/sdk` ^1.29.0
|
|
7
7
|
**Zod:** ^4.4.3
|
|
@@ -56,6 +56,7 @@ Both paths share the same public API. Init copies starter `package.json`, config
|
|
|
56
56
|
| `/storage` | `StorageService` | Storage abstraction |
|
|
57
57
|
| `/storage/types` | `IStorageProvider` | Provider interface |
|
|
58
58
|
| `/canvas` | `DataCanvas`, `CanvasInstance`, `CanvasRegistry`, `IDataCanvasProvider`, `DuckdbProvider`, `spillover`, `inferSchemaFromRows`, `assertReadOnlyQuery`, `quoteIdentifier`, ... | DataCanvas primitive (Tier 3, optional peer dep `@duckdb/node-api`); SQL/analytical workspace + source-agnostic spillover helper |
|
|
59
|
+
| `/mirror` | `defineMirror`, `sqliteMirrorStore`, `buildSchemaSql`, `openSqliteHandle`, `Mirror`, `MirrorStore`, `MirrorDefinition`, `SyncContext`, `SyncPage`, ... | MirrorService primitive (Tier 3, optional peer dep `better-sqlite3` on Node; `bun:sqlite` built-in on Bun); persistent self-refreshing local mirror of a bulk upstream dataset (embedded SQLite + FTS5). Node/Bun only |
|
|
59
60
|
| `/utils` | formatting, encoding, network, pagination, logging, runtime, telemetry, token counting, parsers†, sanitization†, scheduling† | All utilities (†optional peer deps) |
|
|
60
61
|
| `/services` | `OpenRouterProvider`, `SpeechService`, `createSpeechProvider`, `ElevenLabsProvider`, `WhisperProvider`, `GraphService`, provider interfaces and types | LLM, Speech (TTS/STT), Graph services |
|
|
61
62
|
| `/linter` | `validateDefinitions`, `LintReport`, `LintDiagnostic`, `LintInput`, `LintSeverity` | Definition validation |
|
|
@@ -466,7 +467,7 @@ Detailed method signatures, options, and examples live in skill files. Read the
|
|
|
466
467
|
|
|
467
468
|
Each `skills/<name>/SKILL.md` carries `metadata.version` in frontmatter. The `maintenance` skill's Phase A uses this to sync consumer copies — replaces the **entire skill directory** as one unit. Without a version bump, Phase A skips the skill (content-hash backstop catches drift, but noisier).
|
|
468
469
|
|
|
469
|
-
**Policy:** Bump `metadata.version` when changing any file under `skills/<name>/` — SKILL.md is the single version knob for the directory. Typo/whitespace fixes exempt. One bump per release cycle suffices.
|
|
470
|
+
**Policy:** Bump `metadata.version` when changing any file under `skills/<name>/` — SKILL.md is the single version knob for the directory. Typo/whitespace fixes exempt. One bump per release cycle suffices. Enforced by `bun run devcheck` (`scripts/check-skill-versions.ts`): a SKILL.md body change vs `HEAD` without a `metadata.version` bump surfaces as a warning; whitespace-only edits are ignored, and `devcheck.config.json` `skillVersions.ignore` opts out the typo-fix carve-out.
|
|
470
471
|
|
|
471
472
|
Skills live in `skills/<name>/SKILL.md`. Read the relevant skill before starting a task it covers. The full list is discoverable via the agent's skill registry at session start.
|
|
472
473
|
|
package/CLAUDE.md
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# Developer Protocol
|
|
2
2
|
|
|
3
3
|
**Package:** `@cyanheads/mcp-ts-core`
|
|
4
|
-
**Version:** 0.9.
|
|
4
|
+
**Version:** 0.9.18
|
|
5
5
|
**Engines:** Bun ≥1.3.0, Node ≥24.0.0
|
|
6
6
|
**MCP SDK:** `@modelcontextprotocol/sdk` ^1.29.0
|
|
7
7
|
**Zod:** ^4.4.3
|
|
@@ -56,6 +56,7 @@ Both paths share the same public API. Init copies starter `package.json`, config
|
|
|
56
56
|
| `/storage` | `StorageService` | Storage abstraction |
|
|
57
57
|
| `/storage/types` | `IStorageProvider` | Provider interface |
|
|
58
58
|
| `/canvas` | `DataCanvas`, `CanvasInstance`, `CanvasRegistry`, `IDataCanvasProvider`, `DuckdbProvider`, `spillover`, `inferSchemaFromRows`, `assertReadOnlyQuery`, `quoteIdentifier`, ... | DataCanvas primitive (Tier 3, optional peer dep `@duckdb/node-api`); SQL/analytical workspace + source-agnostic spillover helper |
|
|
59
|
+
| `/mirror` | `defineMirror`, `sqliteMirrorStore`, `buildSchemaSql`, `openSqliteHandle`, `Mirror`, `MirrorStore`, `MirrorDefinition`, `SyncContext`, `SyncPage`, ... | MirrorService primitive (Tier 3, optional peer dep `better-sqlite3` on Node; `bun:sqlite` built-in on Bun); persistent self-refreshing local mirror of a bulk upstream dataset (embedded SQLite + FTS5). Node/Bun only |
|
|
59
60
|
| `/utils` | formatting, encoding, network, pagination, logging, runtime, telemetry, token counting, parsers†, sanitization†, scheduling† | All utilities (†optional peer deps) |
|
|
60
61
|
| `/services` | `OpenRouterProvider`, `SpeechService`, `createSpeechProvider`, `ElevenLabsProvider`, `WhisperProvider`, `GraphService`, provider interfaces and types | LLM, Speech (TTS/STT), Graph services |
|
|
61
62
|
| `/linter` | `validateDefinitions`, `LintReport`, `LintDiagnostic`, `LintInput`, `LintSeverity` | Definition validation |
|
|
@@ -466,7 +467,7 @@ Detailed method signatures, options, and examples live in skill files. Read the
|
|
|
466
467
|
|
|
467
468
|
Each `skills/<name>/SKILL.md` carries `metadata.version` in frontmatter. The `maintenance` skill's Phase A uses this to sync consumer copies — replaces the **entire skill directory** as one unit. Without a version bump, Phase A skips the skill (content-hash backstop catches drift, but noisier).
|
|
468
469
|
|
|
469
|
-
**Policy:** Bump `metadata.version` when changing any file under `skills/<name>/` — SKILL.md is the single version knob for the directory. Typo/whitespace fixes exempt. One bump per release cycle suffices.
|
|
470
|
+
**Policy:** Bump `metadata.version` when changing any file under `skills/<name>/` — SKILL.md is the single version knob for the directory. Typo/whitespace fixes exempt. One bump per release cycle suffices. Enforced by `bun run devcheck` (`scripts/check-skill-versions.ts`): a SKILL.md body change vs `HEAD` without a `metadata.version` bump surfaces as a warning; whitespace-only edits are ignored, and `devcheck.config.json` `skillVersions.ignore` opts out the typo-fix carve-out.
|
|
470
471
|
|
|
471
472
|
Skills live in `skills/<name>/SKILL.md`. Read the relevant skill before starting a task it covers. The full list is discoverable via the agent's skill registry at session start.
|
|
472
473
|
|
package/README.md
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
|
|
6
6
|
<div align="center">
|
|
7
7
|
|
|
8
|
-
[](./CHANGELOG.md) [](./LICENSE) [](https://github.com/modelcontextprotocol/modelcontextprotocol/blob/main/docs/specification/2025-11-25/changelog.mdx)
|
|
9
9
|
|
|
10
10
|
[](https://modelcontextprotocol.io/) [](https://www.typescriptlang.org/) [](https://bun.sh/)
|
|
11
11
|
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
---
|
|
2
|
+
summary: "MirrorService — persistent, self-refreshing local mirror of a bulk upstream dataset (embedded SQLite + FTS5) via new @cyanheads/mcp-ts-core/mirror subpath"
|
|
3
|
+
breaking: false
|
|
4
|
+
security: false
|
|
5
|
+
agent-notes: |
|
|
6
|
+
New subpath `/mirror` exports `defineMirror`, `sqliteMirrorStore`, `buildSchemaSql`, `openSqliteHandle`, `Mirror`, `MirrorStore`, `MirrorDefinition`, `SyncContext`, `SyncPage`, and related types. Node/Bun only — unavailable on Workers. Optional peer dep `better-sqlite3` is required on Node; Bun uses the built-in `bun:sqlite`. New skill `api-mirror` covers the full API surface.
|
|
7
|
+
---
|
|
8
|
+
|
|
9
|
+
# 0.9.17 — 2026-05-30
|
|
10
|
+
|
|
11
|
+
## Added
|
|
12
|
+
|
|
13
|
+
- **`defineMirror()`** — assembles a `MirrorStore` + a server `sync` generator into an owned `Mirror` instance with `runSync({ mode: 'init' | 'refresh' })`, `query()`, `status()`, `ready()`, `getByIds()`, and a raw-handle escape hatch (`handle()`). ([#164](https://github.com/cyanheads/mcp-ts-core/issues/164))
|
|
14
|
+
- **`sqliteMirrorStore()`** — embedded-SQLite backend with cross-runtime support (`bun:sqlite` built-in on Bun; `better-sqlite3` optional peer on Node); declarative schema (columns + opt-in FTS5 external-content index with sync triggers + configurable tokenizer + secondary indexes); generic flat `query()` (FTS `MATCH` + equality/range filters + column-or-bm25 sort + pagination); `schema_version` migration runner.
|
|
15
|
+
- **Sync state machine** — volatile `cursor` (intra-run resume token, cleared on completion) kept distinct from the durable `checkpoint` (incremental high-water mark, advances monotonically on success); per-page persistence gives resume-on-interrupt; readiness keys off the durable completion marker so the mirror stays queryable during a refresh.
|
|
16
|
+
- **`MirrorStore` interface** — pluggable backend contract, leaving a clean path to non-SQLite backends without re-architecting.
|
|
17
|
+
- **`@cyanheads/mcp-ts-core/mirror` subpath** — all mirror exports live here; Tier 3 (optional peer dep), Node/Bun only.
|
|
18
|
+
- **`better-sqlite3` optional peer dependency** — required on Node; gated like the existing Supabase/DuckDB providers. Bun uses the built-in `bun:sqlite`.
|
|
19
|
+
- **`api-mirror` skill** — documents the full `/mirror` API surface for downstream agent consumption.
|
|
20
|
+
|
|
21
|
+
## Changed
|
|
22
|
+
|
|
23
|
+
- **`@cloudflare/workers-types`** `4.20260530.1 → 4.20260531.1` (dev) — trivial patch refresh.
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
---
|
|
2
|
+
summary: "Two devcheck enforcement gates: a metadata.version bump check for SKILL.md body changes (#99), and an AST check for open-indexed-named interfaces (#123)."
|
|
3
|
+
breaking: false
|
|
4
|
+
security: false
|
|
5
|
+
---
|
|
6
|
+
|
|
7
|
+
# 0.9.18 — 2026-05-31
|
|
8
|
+
|
|
9
|
+
## Added
|
|
10
|
+
|
|
11
|
+
- **Skill-version enforcement gate** in devcheck — a `skills/<name>/SKILL.md` body change without a `metadata.version` bump surfaces as a warning; whitespace-only edits are ignored; `devcheck.config.json` `skillVersions.ignore` opts out the typo carve-out. ([#99](https://github.com/cyanheads/mcp-ts-core/issues/99))
|
|
12
|
+
- **Open-indexed-named interface gate** in devcheck — an AST check fails on interfaces mixing named members with an open `[key: string]: unknown|any` index signature unless annotated `// allow open-indexed-named: <rationale>`; six framework extensibility bags annotated as the baseline. ([#123](https://github.com/cyanheads/mcp-ts-core/issues/123))
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"toolDefinition.d.ts","sourceRoot":"","sources":["../../../../src/mcp-server/tools/utils/toolDefinition.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oCAAoC,CAAC;AACvE,OAAO,KAAK,EAAE,SAAS,EAAE,WAAW,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAErD,OAAO,KAAK,EAAE,cAAc,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAClE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AAE9D,OAAO,EAAE,KAAK,gBAAgB,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAEzE;;;;GAIG;AACH,MAAM,WAAW,eAAe;IAC9B;;;;OAIG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B;;;;OAIG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB;;;OAGG;IACH,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB;;;OAGG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"toolDefinition.d.ts","sourceRoot":"","sources":["../../../../src/mcp-server/tools/utils/toolDefinition.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oCAAoC,CAAC;AACvE,OAAO,KAAK,EAAE,SAAS,EAAE,WAAW,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAErD,OAAO,KAAK,EAAE,cAAc,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAClE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AAE9D,OAAO,EAAE,KAAK,gBAAgB,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAEzE;;;;GAIG;AACH,MAAM,WAAW,eAAe;IAC9B;;;;OAIG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B;;;;OAIG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB;;;OAGG;IACH,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB;;;OAGG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,MAAM,MAAM,uBAAuB,CAAC,OAAO,SAAS,WAAW,GAAG,SAAS,IACzE,OAAO,SAAS,WAAW,GACvB;KACG,CAAC,IAAI,MAAM,OAAO,CAAC,CAAC,EAAE;QACrB,KAAK,CAAC,EAAE,MAAM,CAAC;QACf;;;;;;;WAOG;QACH,MAAM,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC;KAC7C;CACF,GACD,KAAK,CAAC;AAEZ;;;;;;;;;;;GAWG;AACH,MAAM,WAAW,cAAc,CAC7B,MAAM,SAAS,SAAS,CAAC,WAAW,CAAC,GAAG,SAAS,CAAC,WAAW,CAAC,EAC9D,OAAO,SAAS,SAAS,CAAC,WAAW,CAAC,GAAG,SAAS,CAAC,WAAW,CAAC,EAC/D,OAAO,SAAS,SAAS,aAAa,EAAE,GAAG,SAAS,GAAG,SAAS,EAChE,OAAO,SAAS,WAAW,GAAG,SAAS,GAAG,SAAS;IAEnD,0DAA0D;IAC1D,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAChC,qCAAqC;IACrC,WAAW,CAAC,EAAE,eAAe,CAAC;IAC9B,+EAA+E;IAC/E,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,8BAA8B;IAC9B,WAAW,EAAE,MAAM,CAAC;IACpB;;;;;;;;;;;;;;;;;;;OAmBG;IACH,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB;;;;;;;;;;;OAWG;IACH,iBAAiB,CAAC,EAAE,uBAAuB,CAAC,OAAO,CAAC,CAAC;IACrD;;;;;;;;;;;;;;;;;OAiBG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB;;;;;;;;;;;;;;;;;OAiBG;IACH,MAAM,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,YAAY,EAAE,CAAC;IAClD;;;;;;;;;;;;;;;OAeG;IACH,OAAO,CACL,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,EACtB,GAAG,EAAE,cAAc,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,OAAO,CAAC,GAC9C,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAChD,sEAAsE;IACtE,KAAK,EAAE,MAAM,CAAC;IACd,6CAA6C;IAC7C,IAAI,EAAE,MAAM,CAAC;IACb,uEAAuE;IACvE,MAAM,EAAE,OAAO,CAAC;IAChB;;;;;;OAMG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,qEAAqE;IACrE,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,2CAA2C;IAC3C,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,gEAAgE;AAChE,MAAM,MAAM,iBAAiB,GAAG,cAAc,CAC5C,SAAS,CAAC,WAAW,CAAC,EACtB,SAAS,CAAC,WAAW,CAAC,EACtB,SAAS,aAAa,EAAE,GAAG,SAAS,EACpC,WAAW,GAAG,SAAS,CACxB,CAAC;AAMF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8CG;AACH,wBAAgB,IAAI,CAClB,MAAM,SAAS,SAAS,CAAC,WAAW,CAAC,EACrC,OAAO,SAAS,SAAS,CAAC,WAAW,CAAC,EACtC,KAAK,CAAC,OAAO,SAAS,SAAS,aAAa,EAAE,GAAG,SAAS,GAAG,SAAS,EACtE,KAAK,CAAC,OAAO,SAAS,WAAW,GAAG,SAAS,GAAG,SAAS,EAEzD,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,IAAI,CAAC,cAAc,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,MAAM,CAAC,GACvE,cAAc,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,CAEnD"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"toolDefinition.js","sourceRoot":"","sources":["../../../../src/mcp-server/tools/utils/toolDefinition.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAQH,OAAO,EAAyB,YAAY,EAAE,MAAM,oBAAoB,CAAC;
|
|
1
|
+
{"version":3,"file":"toolDefinition.js","sourceRoot":"","sources":["../../../../src/mcp-server/tools/utils/toolDefinition.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAQH,OAAO,EAAyB,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAgOzE,8EAA8E;AAC9E,UAAU;AACV,8EAA8E;AAE9E;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8CG;AACH,MAAM,UAAU,IAAI,CAMlB,IAAY,EACZ,OAAwE;IAExE,OAAO,EAAE,IAAI,EAAE,GAAG,OAAO,EAAE,CAAC;AAC9B,CAAC"}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview `defineMirror` — assembles a {@link MirrorStore}, a server's
|
|
3
|
+
* `sync` generator, and the runner into a single owned mirror instance. The
|
|
4
|
+
* returned object is what a server holds (one per mirror) and exposes to its
|
|
5
|
+
* tools: `runSync`, `query`, `status`/`ready`, `getByIds`, and the raw handle.
|
|
6
|
+
* @module services/mirror/core/defineMirror
|
|
7
|
+
*/
|
|
8
|
+
import type { MirrorDefinition, MirrorRow, MirrorStatus, MirrorStore, QueryOptions, QueryResult, RunSyncOptions, SqliteHandle, SyncResult } from '../types.js';
|
|
9
|
+
/** Options for a single {@link Mirror.runSync} call. */
|
|
10
|
+
export type MirrorRunOptions = RunSyncOptions & {
|
|
11
|
+
/** Cancels the run; the runner persists state before stopping. Defaults to a never-aborting signal. */
|
|
12
|
+
signal?: AbortSignal;
|
|
13
|
+
};
|
|
14
|
+
/** An owned mirror instance — the return value of {@link defineMirror}. */
|
|
15
|
+
export interface Mirror {
|
|
16
|
+
/** Close the underlying store. */
|
|
17
|
+
close(): Promise<void>;
|
|
18
|
+
/** Fetch records by primary-key list, preserving input order. */
|
|
19
|
+
getByIds(ids: string[]): Promise<MirrorRow[]>;
|
|
20
|
+
/** Stable name from the definition. */
|
|
21
|
+
readonly name: string;
|
|
22
|
+
/** Generic flat query over the mirror. */
|
|
23
|
+
query(options: QueryOptions): Promise<QueryResult>;
|
|
24
|
+
/** The opened runtime-agnostic handle for server-specific access paths. */
|
|
25
|
+
raw(): Promise<SqliteHandle>;
|
|
26
|
+
/** `true` once a full sync has ever completed (queryable even mid-refresh). */
|
|
27
|
+
ready(): Promise<boolean>;
|
|
28
|
+
/** Run a full (`init`) or incremental (`refresh`) sync. */
|
|
29
|
+
runSync(options: MirrorRunOptions): Promise<SyncResult>;
|
|
30
|
+
/** Current sync status, including the durable `ready` marker. */
|
|
31
|
+
status(): Promise<MirrorStatus>;
|
|
32
|
+
/** The backing store (escape hatch for advanced use). */
|
|
33
|
+
readonly store: MirrorStore;
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* Assemble a mirror from a store, a `sync` generator, and a name. Nothing opens
|
|
37
|
+
* until the first call; construction is cheap and side-effect-free.
|
|
38
|
+
*
|
|
39
|
+
* @example
|
|
40
|
+
* ```ts
|
|
41
|
+
* const papers = defineMirror({
|
|
42
|
+
* name: 'arxiv-papers',
|
|
43
|
+
* store: sqliteMirrorStore({
|
|
44
|
+
* path: config.mirrorPath,
|
|
45
|
+
* primaryKey: 'id',
|
|
46
|
+
* columns: { id: 'TEXT', title: 'TEXT', abstract: 'TEXT', updated: 'TEXT' },
|
|
47
|
+
* fts: ['title', 'abstract'],
|
|
48
|
+
* indexes: [{ columns: ['updated'] }],
|
|
49
|
+
* }),
|
|
50
|
+
* async *sync({ mode, cursor, checkpoint, signal }) {
|
|
51
|
+
* for await (const page of harvest({ cursor, since: checkpoint, signal })) {
|
|
52
|
+
* yield { records: page.rows, cursor: page.token, checkpoint: page.maxStamp };
|
|
53
|
+
* }
|
|
54
|
+
* },
|
|
55
|
+
* });
|
|
56
|
+
*
|
|
57
|
+
* await papers.runSync({ mode: 'init', signal: AbortSignal.timeout(3_600_000) });
|
|
58
|
+
* const { rows, total } = await papers.query({ match: 'transformers', limit: 10, offset: 0 });
|
|
59
|
+
* ```
|
|
60
|
+
*/
|
|
61
|
+
export declare function defineMirror(definition: MirrorDefinition): Mirror;
|
|
62
|
+
//# sourceMappingURL=defineMirror.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"defineMirror.d.ts","sourceRoot":"","sources":["../../../../src/services/mirror/core/defineMirror.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAIH,OAAO,KAAK,EACV,gBAAgB,EAEhB,SAAS,EACT,YAAY,EACZ,WAAW,EACX,YAAY,EACZ,WAAW,EACX,cAAc,EACd,YAAY,EACZ,UAAU,EACX,MAAM,aAAa,CAAC;AAGrB,wDAAwD;AACxD,MAAM,MAAM,gBAAgB,GAAG,cAAc,GAAG;IAC9C,uGAAuG;IACvG,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB,CAAC;AAEF,2EAA2E;AAC3E,MAAM,WAAW,MAAM;IACrB,kCAAkC;IAClC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACvB,iEAAiE;IACjE,QAAQ,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC;IAC9C,uCAAuC;IACvC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,0CAA0C;IAC1C,KAAK,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;IACnD,2EAA2E;IAC3E,GAAG,IAAI,OAAO,CAAC,YAAY,CAAC,CAAC;IAC7B,+EAA+E;IAC/E,KAAK,IAAI,OAAO,CAAC,OAAO,CAAC,CAAC;IAC1B,2DAA2D;IAC3D,OAAO,CAAC,OAAO,EAAE,gBAAgB,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;IACxD,iEAAiE;IACjE,MAAM,IAAI,OAAO,CAAC,YAAY,CAAC,CAAC;IAChC,yDAAyD;IACzD,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAC;CAC7B;AAuBD;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,wBAAgB,YAAY,CAAC,UAAU,EAAE,gBAAgB,GAAG,MAAM,CAkDjE"}
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview `defineMirror` — assembles a {@link MirrorStore}, a server's
|
|
3
|
+
* `sync` generator, and the runner into a single owned mirror instance. The
|
|
4
|
+
* returned object is what a server holds (one per mirror) and exposes to its
|
|
5
|
+
* tools: `runSync`, `query`, `status`/`ready`, `getByIds`, and the raw handle.
|
|
6
|
+
* @module services/mirror/core/defineMirror
|
|
7
|
+
*/
|
|
8
|
+
import { logger } from '../../../utils/internal/logger.js';
|
|
9
|
+
import { requestContextService } from '../../../utils/internal/requestContext.js';
|
|
10
|
+
import { runSync as runSyncCore } from './runner.js';
|
|
11
|
+
/**
|
|
12
|
+
* Forwards sync logs to the framework logger when the definition supplies none.
|
|
13
|
+
* Sync runs outside the request pipeline, so each call wraps `meta` in a fresh
|
|
14
|
+
* `RequestContext` (the shape the framework logger expects); `error` takes a
|
|
15
|
+
* `RequestContext` in the no-Error overload.
|
|
16
|
+
*/
|
|
17
|
+
const defaultLogger = {
|
|
18
|
+
debug: (message, meta) => logger.debug(message, ctxFrom(meta)),
|
|
19
|
+
info: (message, meta) => logger.info(message, ctxFrom(meta)),
|
|
20
|
+
notice: (message, meta) => logger.notice(message, ctxFrom(meta)),
|
|
21
|
+
warning: (message, meta) => logger.warning(message, ctxFrom(meta)),
|
|
22
|
+
error: (message, meta) => logger.error(message, ctxFrom(meta)),
|
|
23
|
+
};
|
|
24
|
+
function ctxFrom(meta) {
|
|
25
|
+
return requestContextService.createRequestContext({
|
|
26
|
+
operation: 'mirror.sync',
|
|
27
|
+
...(meta ?? {}),
|
|
28
|
+
});
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Assemble a mirror from a store, a `sync` generator, and a name. Nothing opens
|
|
32
|
+
* until the first call; construction is cheap and side-effect-free.
|
|
33
|
+
*
|
|
34
|
+
* @example
|
|
35
|
+
* ```ts
|
|
36
|
+
* const papers = defineMirror({
|
|
37
|
+
* name: 'arxiv-papers',
|
|
38
|
+
* store: sqliteMirrorStore({
|
|
39
|
+
* path: config.mirrorPath,
|
|
40
|
+
* primaryKey: 'id',
|
|
41
|
+
* columns: { id: 'TEXT', title: 'TEXT', abstract: 'TEXT', updated: 'TEXT' },
|
|
42
|
+
* fts: ['title', 'abstract'],
|
|
43
|
+
* indexes: [{ columns: ['updated'] }],
|
|
44
|
+
* }),
|
|
45
|
+
* async *sync({ mode, cursor, checkpoint, signal }) {
|
|
46
|
+
* for await (const page of harvest({ cursor, since: checkpoint, signal })) {
|
|
47
|
+
* yield { records: page.rows, cursor: page.token, checkpoint: page.maxStamp };
|
|
48
|
+
* }
|
|
49
|
+
* },
|
|
50
|
+
* });
|
|
51
|
+
*
|
|
52
|
+
* await papers.runSync({ mode: 'init', signal: AbortSignal.timeout(3_600_000) });
|
|
53
|
+
* const { rows, total } = await papers.query({ match: 'transformers', limit: 10, offset: 0 });
|
|
54
|
+
* ```
|
|
55
|
+
*/
|
|
56
|
+
export function defineMirror(definition) {
|
|
57
|
+
const { name, store, sync } = definition;
|
|
58
|
+
const log = definition.logger ?? defaultLogger;
|
|
59
|
+
return {
|
|
60
|
+
name,
|
|
61
|
+
store,
|
|
62
|
+
runSync(options) {
|
|
63
|
+
const signal = options.signal ?? new AbortController().signal;
|
|
64
|
+
const runOptions = {
|
|
65
|
+
mode: options.mode,
|
|
66
|
+
...(options.onProgress && { onProgress: options.onProgress }),
|
|
67
|
+
};
|
|
68
|
+
return runSyncCore(store, sync, { log, signal }, runOptions);
|
|
69
|
+
},
|
|
70
|
+
query(options) {
|
|
71
|
+
return store.query(options);
|
|
72
|
+
},
|
|
73
|
+
getByIds(ids) {
|
|
74
|
+
return store.getByIds(ids);
|
|
75
|
+
},
|
|
76
|
+
async status() {
|
|
77
|
+
const state = await store.readState();
|
|
78
|
+
return {
|
|
79
|
+
status: state.status,
|
|
80
|
+
ready: state.completedAt != null,
|
|
81
|
+
...(state.checkpoint !== undefined && { checkpoint: state.checkpoint }),
|
|
82
|
+
...(state.total !== undefined && { total: state.total }),
|
|
83
|
+
...(state.startedAt !== undefined && { startedAt: state.startedAt }),
|
|
84
|
+
...(state.completedAt !== undefined && { completedAt: state.completedAt }),
|
|
85
|
+
...(state.error !== undefined && { error: state.error }),
|
|
86
|
+
};
|
|
87
|
+
},
|
|
88
|
+
async ready() {
|
|
89
|
+
return (await store.readState()).completedAt != null;
|
|
90
|
+
},
|
|
91
|
+
raw() {
|
|
92
|
+
return store.raw();
|
|
93
|
+
},
|
|
94
|
+
close() {
|
|
95
|
+
return store.close();
|
|
96
|
+
},
|
|
97
|
+
};
|
|
98
|
+
}
|
|
99
|
+
//# sourceMappingURL=defineMirror.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"defineMirror.js","sourceRoot":"","sources":["../../../../src/services/mirror/core/defineMirror.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,MAAM,EAAE,MAAM,4BAA4B,CAAC;AACpD,OAAO,EAAE,qBAAqB,EAAE,MAAM,oCAAoC,CAAC;AAa3E,OAAO,EAAE,OAAO,IAAI,WAAW,EAAE,MAAM,aAAa,CAAC;AA8BrD;;;;;GAKG;AACH,MAAM,aAAa,GAAiB;IAClC,KAAK,EAAE,CAAC,OAAO,EAAE,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;IAC9D,IAAI,EAAE,CAAC,OAAO,EAAE,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5D,MAAM,EAAE,CAAC,OAAO,EAAE,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;IAChE,OAAO,EAAE,CAAC,OAAO,EAAE,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;IAClE,KAAK,EAAE,CAAC,OAAO,EAAE,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;CAC/D,CAAC;AAEF,SAAS,OAAO,CAAC,IAAa;IAC5B,OAAO,qBAAqB,CAAC,oBAAoB,CAAC;QAChD,SAAS,EAAE,aAAa;QACxB,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC;KAChB,CAAC,CAAC;AACL,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,MAAM,UAAU,YAAY,CAAC,UAA4B;IACvD,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,UAAU,CAAC;IACzC,MAAM,GAAG,GAAG,UAAU,CAAC,MAAM,IAAI,aAAa,CAAC;IAE/C,OAAO;QACL,IAAI;QACJ,KAAK;QAEL,OAAO,CAAC,OAAyB;YAC/B,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,IAAI,eAAe,EAAE,CAAC,MAAM,CAAC;YAC9D,MAAM,UAAU,GAAmB;gBACjC,IAAI,EAAE,OAAO,CAAC,IAAI;gBAClB,GAAG,CAAC,OAAO,CAAC,UAAU,IAAI,EAAE,UAAU,EAAE,OAAO,CAAC,UAAU,EAAE,CAAC;aAC9D,CAAC;YACF,OAAO,WAAW,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,UAAU,CAAC,CAAC;QAC/D,CAAC;QAED,KAAK,CAAC,OAAqB;YACzB,OAAO,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC9B,CAAC;QAED,QAAQ,CAAC,GAAa;YACpB,OAAO,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;QAC7B,CAAC;QAED,KAAK,CAAC,MAAM;YACV,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,SAAS,EAAE,CAAC;YACtC,OAAO;gBACL,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,KAAK,EAAE,KAAK,CAAC,WAAW,IAAI,IAAI;gBAChC,GAAG,CAAC,KAAK,CAAC,UAAU,KAAK,SAAS,IAAI,EAAE,UAAU,EAAE,KAAK,CAAC,UAAU,EAAE,CAAC;gBACvE,GAAG,CAAC,KAAK,CAAC,KAAK,KAAK,SAAS,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC;gBACxD,GAAG,CAAC,KAAK,CAAC,SAAS,KAAK,SAAS,IAAI,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,CAAC;gBACpE,GAAG,CAAC,KAAK,CAAC,WAAW,KAAK,SAAS,IAAI,EAAE,WAAW,EAAE,KAAK,CAAC,WAAW,EAAE,CAAC;gBAC1E,GAAG,CAAC,KAAK,CAAC,KAAK,KAAK,SAAS,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC;aACzD,CAAC;QACJ,CAAC;QAED,KAAK,CAAC,KAAK;YACT,OAAO,CAAC,MAAM,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC,WAAW,IAAI,IAAI,CAAC;QACvD,CAAC;QAED,GAAG;YACD,OAAO,KAAK,CAAC,GAAG,EAAE,CAAC;QACrB,CAAC;QAED,KAAK;YACH,OAAO,KAAK,CAAC,KAAK,EAAE,CAAC;QACvB,CAAC;KACF,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview Sync runner — drives a server's `sync` generator against a
|
|
3
|
+
* {@link MirrorStore} for full init and incremental refresh. Owns the
|
|
4
|
+
* cursor/checkpoint state machine, per-page persistence, resume-on-interrupt,
|
|
5
|
+
* and terminal status transitions.
|
|
6
|
+
* @module services/mirror/core/runner
|
|
7
|
+
*/
|
|
8
|
+
import type { MirrorLogger, MirrorStore, RunSyncOptions, SyncGenerator, SyncResult } from '../types.js';
|
|
9
|
+
/** Minimal context the runner consumes — a duck-typed logger plus cancellation. */
|
|
10
|
+
export interface RunnerContext {
|
|
11
|
+
log: MirrorLogger;
|
|
12
|
+
signal: AbortSignal;
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* Run one sync cycle.
|
|
16
|
+
*
|
|
17
|
+
* - `init` performs a full harvest. With no prior state it starts from scratch;
|
|
18
|
+
* if a previous init left an incomplete dataset (status `in_progress` from a
|
|
19
|
+
* crash, or `error` from a caught failure) and a `cursor` was persisted, it
|
|
20
|
+
* resumes from that cursor, with `checkpoint` available for recovery.
|
|
21
|
+
* - `refresh` performs an incremental harvest from the durable `checkpoint`.
|
|
22
|
+
*
|
|
23
|
+
* State is persisted after every page, so an interrupt at any point resumes from
|
|
24
|
+
* the last yielded page. On success the volatile `cursor` is cleared and
|
|
25
|
+
* `completedAt`/`total` advance; on failure they are preserved (a failed
|
|
26
|
+
* refresh on top of a complete mirror stays "ready").
|
|
27
|
+
*/
|
|
28
|
+
export declare function runSync(store: MirrorStore, sync: SyncGenerator, ctx: RunnerContext, options: RunSyncOptions): Promise<SyncResult>;
|
|
29
|
+
//# sourceMappingURL=runner.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../../../src/services/mirror/core/runner.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EACV,YAAY,EACZ,WAAW,EACX,cAAc,EACd,aAAa,EACb,UAAU,EAEX,MAAM,aAAa,CAAC;AAErB,mFAAmF;AACnF,MAAM,WAAW,aAAa;IAC5B,GAAG,EAAE,YAAY,CAAC;IAClB,MAAM,EAAE,WAAW,CAAC;CACrB;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAsB,OAAO,CAC3B,KAAK,EAAE,WAAW,EAClB,IAAI,EAAE,aAAa,EACnB,GAAG,EAAE,aAAa,EAClB,OAAO,EAAE,cAAc,GACtB,OAAO,CAAC,UAAU,CAAC,CAiFrB"}
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview Sync runner — drives a server's `sync` generator against a
|
|
3
|
+
* {@link MirrorStore} for full init and incremental refresh. Owns the
|
|
4
|
+
* cursor/checkpoint state machine, per-page persistence, resume-on-interrupt,
|
|
5
|
+
* and terminal status transitions.
|
|
6
|
+
* @module services/mirror/core/runner
|
|
7
|
+
*/
|
|
8
|
+
/**
|
|
9
|
+
* Run one sync cycle.
|
|
10
|
+
*
|
|
11
|
+
* - `init` performs a full harvest. With no prior state it starts from scratch;
|
|
12
|
+
* if a previous init left an incomplete dataset (status `in_progress` from a
|
|
13
|
+
* crash, or `error` from a caught failure) and a `cursor` was persisted, it
|
|
14
|
+
* resumes from that cursor, with `checkpoint` available for recovery.
|
|
15
|
+
* - `refresh` performs an incremental harvest from the durable `checkpoint`.
|
|
16
|
+
*
|
|
17
|
+
* State is persisted after every page, so an interrupt at any point resumes from
|
|
18
|
+
* the last yielded page. On success the volatile `cursor` is cleared and
|
|
19
|
+
* `completedAt`/`total` advance; on failure they are preserved (a failed
|
|
20
|
+
* refresh on top of a complete mirror stays "ready").
|
|
21
|
+
*/
|
|
22
|
+
export async function runSync(store, sync, ctx, options) {
|
|
23
|
+
const existing = await store.readState();
|
|
24
|
+
// An init re-runs from scratch only when the prior run completed; otherwise it
|
|
25
|
+
// resumes from a persisted cursor (crash mid-write, or caught error mid-init).
|
|
26
|
+
const incompleteInit = options.mode === 'init' && existing.status !== 'complete';
|
|
27
|
+
const resuming = incompleteInit && existing.cursor !== undefined;
|
|
28
|
+
const startCursor = resuming ? existing.cursor : undefined;
|
|
29
|
+
const startCheckpoint = options.mode === 'refresh' || incompleteInit ? existing.checkpoint : undefined;
|
|
30
|
+
const startedAt = (resuming && existing.startedAt) || new Date().toISOString();
|
|
31
|
+
let cursor = startCursor;
|
|
32
|
+
let checkpoint = startCheckpoint;
|
|
33
|
+
await store.writeState({ status: 'in_progress', startedAt, cursor, checkpoint });
|
|
34
|
+
let pagesFetched = 0;
|
|
35
|
+
let recordsApplied = 0;
|
|
36
|
+
let tombstonesApplied = 0;
|
|
37
|
+
try {
|
|
38
|
+
for await (const page of sync({
|
|
39
|
+
mode: options.mode,
|
|
40
|
+
cursor: startCursor,
|
|
41
|
+
checkpoint: startCheckpoint,
|
|
42
|
+
signal: ctx.signal,
|
|
43
|
+
})) {
|
|
44
|
+
const tombstones = page.tombstones ?? [];
|
|
45
|
+
await store.applyBatch(page.records, tombstones);
|
|
46
|
+
pagesFetched += 1;
|
|
47
|
+
recordsApplied += page.records.length;
|
|
48
|
+
tombstonesApplied += tombstones.length;
|
|
49
|
+
cursor = page.cursor;
|
|
50
|
+
// Durable high-water mark advances monotonically (lexicographic compare).
|
|
51
|
+
if (page.checkpoint && (!checkpoint || page.checkpoint > checkpoint)) {
|
|
52
|
+
checkpoint = page.checkpoint;
|
|
53
|
+
}
|
|
54
|
+
await store.writeState({ status: 'in_progress', startedAt, cursor, checkpoint });
|
|
55
|
+
options.onProgress?.({
|
|
56
|
+
pages: pagesFetched,
|
|
57
|
+
records: recordsApplied,
|
|
58
|
+
tombstones: tombstonesApplied,
|
|
59
|
+
cursor,
|
|
60
|
+
checkpoint,
|
|
61
|
+
});
|
|
62
|
+
}
|
|
63
|
+
const total = await store.count();
|
|
64
|
+
await store.writeState({
|
|
65
|
+
status: 'complete',
|
|
66
|
+
startedAt,
|
|
67
|
+
checkpoint,
|
|
68
|
+
cursor: undefined, // volatile cursor is meaningless once complete
|
|
69
|
+
completedAt: new Date().toISOString(),
|
|
70
|
+
total,
|
|
71
|
+
});
|
|
72
|
+
ctx.log.info?.('Mirror sync complete', {
|
|
73
|
+
mode: options.mode,
|
|
74
|
+
pagesFetched,
|
|
75
|
+
recordsApplied,
|
|
76
|
+
tombstonesApplied,
|
|
77
|
+
total,
|
|
78
|
+
});
|
|
79
|
+
return { pagesFetched, recordsApplied, tombstonesApplied, total };
|
|
80
|
+
}
|
|
81
|
+
catch (err) {
|
|
82
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
83
|
+
const errorState = {
|
|
84
|
+
status: 'error',
|
|
85
|
+
startedAt,
|
|
86
|
+
cursor,
|
|
87
|
+
checkpoint,
|
|
88
|
+
error: message,
|
|
89
|
+
};
|
|
90
|
+
await store.writeState(errorState);
|
|
91
|
+
ctx.log.error?.('Mirror sync failed', { mode: options.mode, error: message });
|
|
92
|
+
throw err;
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
//# sourceMappingURL=runner.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"runner.js","sourceRoot":"","sources":["../../../../src/services/mirror/core/runner.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAiBH;;;;;;;;;;;;;GAaG;AACH,MAAM,CAAC,KAAK,UAAU,OAAO,CAC3B,KAAkB,EAClB,IAAmB,EACnB,GAAkB,EAClB,OAAuB;IAEvB,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,SAAS,EAAE,CAAC;IACzC,+EAA+E;IAC/E,+EAA+E;IAC/E,MAAM,cAAc,GAAG,OAAO,CAAC,IAAI,KAAK,MAAM,IAAI,QAAQ,CAAC,MAAM,KAAK,UAAU,CAAC;IACjF,MAAM,QAAQ,GAAG,cAAc,IAAI,QAAQ,CAAC,MAAM,KAAK,SAAS,CAAC;IAEjE,MAAM,WAAW,GAAG,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;IAC3D,MAAM,eAAe,GACnB,OAAO,CAAC,IAAI,KAAK,SAAS,IAAI,cAAc,CAAC,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC;IACjF,MAAM,SAAS,GAAG,CAAC,QAAQ,IAAI,QAAQ,CAAC,SAAS,CAAC,IAAI,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAE/E,IAAI,MAAM,GAAG,WAAW,CAAC;IACzB,IAAI,UAAU,GAAG,eAAe,CAAC;IAEjC,MAAM,KAAK,CAAC,UAAU,CAAC,EAAE,MAAM,EAAE,aAAa,EAAE,SAAS,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,CAAC;IAEjF,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,cAAc,GAAG,CAAC,CAAC;IACvB,IAAI,iBAAiB,GAAG,CAAC,CAAC;IAE1B,IAAI,CAAC;QACH,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,IAAI,CAAC;YAC5B,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,MAAM,EAAE,WAAW;YACnB,UAAU,EAAE,eAAe;YAC3B,MAAM,EAAE,GAAG,CAAC,MAAM;SACnB,CAAC,EAAE,CAAC;YACH,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,EAAE,CAAC;YACzC,MAAM,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;YAEjD,YAAY,IAAI,CAAC,CAAC;YAClB,cAAc,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;YACtC,iBAAiB,IAAI,UAAU,CAAC,MAAM,CAAC;YAEvC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;YACrB,0EAA0E;YAC1E,IAAI,IAAI,CAAC,UAAU,IAAI,CAAC,CAAC,UAAU,IAAI,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC,EAAE,CAAC;gBACrE,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;YAC/B,CAAC;YAED,MAAM,KAAK,CAAC,UAAU,CAAC,EAAE,MAAM,EAAE,aAAa,EAAE,SAAS,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,CAAC;YACjF,OAAO,CAAC,UAAU,EAAE,CAAC;gBACnB,KAAK,EAAE,YAAY;gBACnB,OAAO,EAAE,cAAc;gBACvB,UAAU,EAAE,iBAAiB;gBAC7B,MAAM;gBACN,UAAU;aACX,CAAC,CAAC;QACL,CAAC;QAED,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;QAClC,MAAM,KAAK,CAAC,UAAU,CAAC;YACrB,MAAM,EAAE,UAAU;YAClB,SAAS;YACT,UAAU;YACV,MAAM,EAAE,SAAS,EAAE,+CAA+C;YAClE,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACrC,KAAK;SACN,CAAC,CAAC;QACH,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,sBAAsB,EAAE;YACrC,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,YAAY;YACZ,cAAc;YACd,iBAAiB;YACjB,KAAK;SACN,CAAC,CAAC;QACH,OAAO,EAAE,YAAY,EAAE,cAAc,EAAE,iBAAiB,EAAE,KAAK,EAAE,CAAC;IACpE,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACjE,MAAM,UAAU,GAAc;YAC5B,MAAM,EAAE,OAAO;YACf,SAAS;YACT,MAAM;YACN,UAAU;YACV,KAAK,EAAE,OAAO;SACf,CAAC;QACF,MAAM,KAAK,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;QACnC,GAAG,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,oBAAoB,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC;QAC9E,MAAM,GAAG,CAAC;IACZ,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview Public barrel for the MirrorService primitive. Servers import
|
|
3
|
+
* from `@cyanheads/mcp-ts-core/mirror` to stand up a persistent, self-refreshing
|
|
4
|
+
* local mirror of a bulk upstream dataset: declare a store + a `sync` generator
|
|
5
|
+
* via {@link defineMirror}, then `runSync` / `query` / `status`.
|
|
6
|
+
*
|
|
7
|
+
* The SQLite driver is loaded lazily on first use, so importing this barrel does
|
|
8
|
+
* not pull in `bun:sqlite` / `better-sqlite3` until a mirror is actually opened.
|
|
9
|
+
*
|
|
10
|
+
* @module services/mirror
|
|
11
|
+
*/
|
|
12
|
+
export { defineMirror, type Mirror, type MirrorRunOptions } from './core/defineMirror.js';
|
|
13
|
+
export { type RunnerContext, runSync } from './core/runner.js';
|
|
14
|
+
export { type OpenHandleOptions, openSqliteHandle, type SqliteHandle, type SqliteStatement, type SqlValue, } from './sqlite/handle.js';
|
|
15
|
+
export { buildSchemaSql, DEFAULT_FTS_TOKENIZER, type SchemaSpec, validateSchemaSpec, } from './sqlite/schema.js';
|
|
16
|
+
export { type SqliteMirrorStoreSpec, sqliteMirrorStore, } from './sqlite/sqliteMirrorStore.js';
|
|
17
|
+
export type { FilterOp, Migration, MirrorDefinition, MirrorLogger, MirrorRow, MirrorStatus, MirrorStore, QueryFilter, QueryOptions, QueryResult, QuerySort, RunSyncOptions, SyncContext, SyncGenerator, SyncMode, SyncPage, SyncProgress, SyncResult, SyncState, SyncStatus, } from './types.js';
|
|
18
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/services/mirror/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAAE,YAAY,EAAE,KAAK,MAAM,EAAE,KAAK,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC1F,OAAO,EAAE,KAAK,aAAa,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAC/D,OAAO,EACL,KAAK,iBAAiB,EACtB,gBAAgB,EAChB,KAAK,YAAY,EACjB,KAAK,eAAe,EACpB,KAAK,QAAQ,GACd,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EACL,cAAc,EACd,qBAAqB,EACrB,KAAK,UAAU,EACf,kBAAkB,GACnB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EACL,KAAK,qBAAqB,EAC1B,iBAAiB,GAClB,MAAM,+BAA+B,CAAC;AACvC,YAAY,EACV,QAAQ,EACR,SAAS,EACT,gBAAgB,EAChB,YAAY,EACZ,SAAS,EACT,YAAY,EACZ,WAAW,EACX,WAAW,EACX,YAAY,EACZ,WAAW,EACX,SAAS,EACT,cAAc,EACd,WAAW,EACX,aAAa,EACb,QAAQ,EACR,QAAQ,EACR,YAAY,EACZ,UAAU,EACV,SAAS,EACT,UAAU,GACX,MAAM,YAAY,CAAC"}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview Public barrel for the MirrorService primitive. Servers import
|
|
3
|
+
* from `@cyanheads/mcp-ts-core/mirror` to stand up a persistent, self-refreshing
|
|
4
|
+
* local mirror of a bulk upstream dataset: declare a store + a `sync` generator
|
|
5
|
+
* via {@link defineMirror}, then `runSync` / `query` / `status`.
|
|
6
|
+
*
|
|
7
|
+
* The SQLite driver is loaded lazily on first use, so importing this barrel does
|
|
8
|
+
* not pull in `bun:sqlite` / `better-sqlite3` until a mirror is actually opened.
|
|
9
|
+
*
|
|
10
|
+
* @module services/mirror
|
|
11
|
+
*/
|
|
12
|
+
export { defineMirror } from './core/defineMirror.js';
|
|
13
|
+
export { runSync } from './core/runner.js';
|
|
14
|
+
export { openSqliteHandle, } from './sqlite/handle.js';
|
|
15
|
+
export { buildSchemaSql, DEFAULT_FTS_TOKENIZER, validateSchemaSpec, } from './sqlite/schema.js';
|
|
16
|
+
export { sqliteMirrorStore, } from './sqlite/sqliteMirrorStore.js';
|
|
17
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/services/mirror/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAAE,YAAY,EAAsC,MAAM,wBAAwB,CAAC;AAC1F,OAAO,EAAsB,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAC/D,OAAO,EAEL,gBAAgB,GAIjB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EACL,cAAc,EACd,qBAAqB,EAErB,kBAAkB,GACnB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAEL,iBAAiB,GAClB,MAAM,+BAA+B,CAAC"}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview Runtime-agnostic SQLite handle for the mirror store. Uses the
|
|
3
|
+
* built-in `bun:sqlite` driver under Bun and the `better-sqlite3` optional peer
|
|
4
|
+
* dependency on Node — both exposed through one synchronous handle interface
|
|
5
|
+
* (the intersection of the two driver APIs).
|
|
6
|
+
*
|
|
7
|
+
* The drivers are loaded via variable-specifier dynamic imports so the
|
|
8
|
+
* framework typechecks and builds without `bun-types` in scope or
|
|
9
|
+
* `better-sqlite3` installed; both resolve at runtime on the matching runtime.
|
|
10
|
+
* @module services/mirror/sqlite/handle
|
|
11
|
+
*/
|
|
12
|
+
/** Primitive value storable in a mirror column. Buffers/bigints are out of scope for v1. */
|
|
13
|
+
export type SqlValue = string | number | null;
|
|
14
|
+
/**
|
|
15
|
+
* Runtime-agnostic prepared statement — the intersection of the `bun:sqlite`
|
|
16
|
+
* and `better-sqlite3` statement APIs. Bound parameters are passed positionally.
|
|
17
|
+
*/
|
|
18
|
+
export interface SqliteStatement<TRow = unknown> {
|
|
19
|
+
all(...params: SqlValue[]): TRow[];
|
|
20
|
+
get(...params: SqlValue[]): TRow | undefined;
|
|
21
|
+
run(...params: SqlValue[]): {
|
|
22
|
+
changes: number;
|
|
23
|
+
lastInsertRowid: number | bigint;
|
|
24
|
+
};
|
|
25
|
+
}
|
|
26
|
+
/** Runtime-agnostic database handle. Synchronous — both drivers are synchronous. */
|
|
27
|
+
export interface SqliteHandle {
|
|
28
|
+
close(): void;
|
|
29
|
+
exec(sql: string): void;
|
|
30
|
+
prepare<TRow = unknown>(sql: string): SqliteStatement<TRow>;
|
|
31
|
+
transaction<T>(fn: () => T): T;
|
|
32
|
+
}
|
|
33
|
+
/** Options for {@link openSqliteHandle}. */
|
|
34
|
+
export interface OpenHandleOptions {
|
|
35
|
+
/** `PRAGMA busy_timeout` in ms — how long a writer waits on a locked DB. Default 5000. */
|
|
36
|
+
busyTimeoutMs?: number;
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* Open (or create) a SQLite database at `path`, picking the driver for the
|
|
40
|
+
* current runtime. Creates the parent directory, enables WAL, and sets
|
|
41
|
+
* `busy_timeout` so a refresh writer and reader processes coexist without
|
|
42
|
+
* spurious `database is locked` errors.
|
|
43
|
+
*
|
|
44
|
+
* Throws `ConfigurationError` on Node when `better-sqlite3` is not installed,
|
|
45
|
+
* and `DatabaseError` for any other open failure.
|
|
46
|
+
*/
|
|
47
|
+
export declare function openSqliteHandle(path: string, options?: OpenHandleOptions): Promise<SqliteHandle>;
|
|
48
|
+
//# sourceMappingURL=handle.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"handle.d.ts","sourceRoot":"","sources":["../../../../src/services/mirror/sqlite/handle.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAOH,4FAA4F;AAC5F,MAAM,MAAM,QAAQ,GAAG,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC;AAE9C;;;GAGG;AACH,MAAM,WAAW,eAAe,CAAC,IAAI,GAAG,OAAO;IAC7C,GAAG,CAAC,GAAG,MAAM,EAAE,QAAQ,EAAE,GAAG,IAAI,EAAE,CAAC;IACnC,GAAG,CAAC,GAAG,MAAM,EAAE,QAAQ,EAAE,GAAG,IAAI,GAAG,SAAS,CAAC;IAC7C,GAAG,CAAC,GAAG,MAAM,EAAE,QAAQ,EAAE,GAAG;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,eAAe,EAAE,MAAM,GAAG,MAAM,CAAA;KAAE,CAAC;CACnF;AAED,oFAAoF;AACpF,MAAM,WAAW,YAAY;IAC3B,KAAK,IAAI,IAAI,CAAC;IACd,IAAI,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,OAAO,CAAC,IAAI,GAAG,OAAO,EAAE,GAAG,EAAE,MAAM,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IAC5D,WAAW,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC;CAChC;AAED,4CAA4C;AAC5C,MAAM,WAAW,iBAAiB;IAChC,0FAA0F;IAC1F,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AA0CD;;;;;;;;GAQG;AACH,wBAAsB,gBAAgB,CACpC,IAAI,EAAE,MAAM,EACZ,OAAO,GAAE,iBAAsB,GAC9B,OAAO,CAAC,YAAY,CAAC,CAuBvB"}
|