ai-database 2.1.3 → 2.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +35 -1
- package/README.md +880 -669
- package/dist/actions.d.ts +2 -2
- package/dist/actions.d.ts.map +1 -1
- package/dist/actions.js +1 -1
- package/dist/actions.js.map +1 -1
- package/dist/ai-promise-db.d.ts +49 -23
- package/dist/ai-promise-db.d.ts.map +1 -1
- package/dist/ai-promise-db.js +91 -63
- package/dist/ai-promise-db.js.map +1 -1
- package/dist/authorization.d.ts.map +1 -1
- package/dist/authorization.js +38 -30
- package/dist/authorization.js.map +1 -1
- package/dist/cascade-orchestrator.d.ts +404 -0
- package/dist/cascade-orchestrator.d.ts.map +1 -0
- package/dist/cascade-orchestrator.js +828 -0
- package/dist/cascade-orchestrator.js.map +1 -0
- package/dist/cascade-write-strategy.d.ts +584 -0
- package/dist/cascade-write-strategy.d.ts.map +1 -0
- package/dist/cascade-write-strategy.js +590 -0
- package/dist/cascade-write-strategy.js.map +1 -0
- package/dist/ch-adapter.d.ts +358 -0
- package/dist/ch-adapter.d.ts.map +1 -0
- package/dist/ch-adapter.js +929 -0
- package/dist/ch-adapter.js.map +1 -0
- package/dist/client/index.d.ts +42 -0
- package/dist/client/index.d.ts.map +1 -0
- package/dist/client/index.js +43 -0
- package/dist/client/index.js.map +1 -0
- package/dist/client.d.ts +266 -0
- package/dist/client.d.ts.map +1 -0
- package/dist/client.js +81 -0
- package/dist/client.js.map +1 -0
- package/dist/constants.d.ts +64 -1
- package/dist/constants.d.ts.map +1 -1
- package/dist/constants.js +52 -2
- package/dist/constants.js.map +1 -1
- package/dist/dataloader.d.ts +99 -0
- package/dist/dataloader.d.ts.map +1 -0
- package/dist/dataloader.js +225 -0
- package/dist/dataloader.js.map +1 -0
- package/dist/db-provider-port.d.ts +501 -0
- package/dist/db-provider-port.d.ts.map +1 -0
- package/dist/db-provider-port.js +113 -0
- package/dist/db-provider-port.js.map +1 -0
- package/dist/digital-objects-provider.d.ts +49 -0
- package/dist/digital-objects-provider.d.ts.map +1 -0
- package/dist/digital-objects-provider.js +55 -0
- package/dist/digital-objects-provider.js.map +1 -0
- package/dist/do-sqlite-adapter.d.ts +402 -0
- package/dist/do-sqlite-adapter.d.ts.map +1 -0
- package/dist/do-sqlite-adapter.js +745 -0
- package/dist/do-sqlite-adapter.js.map +1 -0
- package/dist/docs-rels/custom-types.d.ts +134 -0
- package/dist/docs-rels/custom-types.d.ts.map +1 -0
- package/dist/docs-rels/custom-types.js +70 -0
- package/dist/docs-rels/custom-types.js.map +1 -0
- package/dist/docs-rels/index.d.ts +16 -0
- package/dist/docs-rels/index.d.ts.map +1 -0
- package/dist/docs-rels/index.js +16 -0
- package/dist/docs-rels/index.js.map +1 -0
- package/dist/docs-rels/migrations/index.d.ts +30 -0
- package/dist/docs-rels/migrations/index.d.ts.map +1 -0
- package/dist/docs-rels/migrations/index.js +128 -0
- package/dist/docs-rels/migrations/index.js.map +1 -0
- package/dist/docs-rels/schema.d.ts +2961 -0
- package/dist/docs-rels/schema.d.ts.map +1 -0
- package/dist/docs-rels/schema.js +244 -0
- package/dist/docs-rels/schema.js.map +1 -0
- package/dist/durable-clickhouse.d.ts.map +1 -1
- package/dist/durable-clickhouse.js +16 -13
- package/dist/durable-clickhouse.js.map +1 -1
- package/dist/durable-promise.d.ts.map +1 -1
- package/dist/durable-promise.js +34 -15
- package/dist/durable-promise.js.map +1 -1
- package/dist/errors.d.ts +127 -0
- package/dist/errors.d.ts.map +1 -0
- package/dist/errors.js +210 -0
- package/dist/errors.js.map +1 -0
- package/dist/eventbridge.d.ts +117 -0
- package/dist/eventbridge.d.ts.map +1 -0
- package/dist/eventbridge.js +238 -0
- package/dist/eventbridge.js.map +1 -0
- package/dist/events.d.ts +2 -2
- package/dist/events.d.ts.map +1 -1
- package/dist/events.js +1 -1
- package/dist/events.js.map +1 -1
- package/dist/execution-queue.d.ts.map +1 -1
- package/dist/execution-queue.js +4 -5
- package/dist/execution-queue.js.map +1 -1
- package/dist/index.d.ts +35 -8
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +106 -6
- package/dist/index.js.map +1 -1
- package/dist/linguistic.d.ts +3 -108
- package/dist/linguistic.d.ts.map +1 -1
- package/dist/linguistic.js +3 -372
- package/dist/linguistic.js.map +1 -1
- package/dist/logger.d.ts +132 -0
- package/dist/logger.d.ts.map +1 -0
- package/dist/logger.js +137 -0
- package/dist/logger.js.map +1 -0
- package/dist/memory-provider.d.ts +128 -0
- package/dist/memory-provider.d.ts.map +1 -1
- package/dist/memory-provider.js +592 -257
- package/dist/memory-provider.js.map +1 -1
- package/dist/pg-adapter.d.ts +424 -0
- package/dist/pg-adapter.d.ts.map +1 -0
- package/dist/pg-adapter.js +921 -0
- package/dist/pg-adapter.js.map +1 -0
- package/dist/pipelines-iceberg-emitter.d.ts +327 -0
- package/dist/pipelines-iceberg-emitter.d.ts.map +1 -0
- package/dist/pipelines-iceberg-emitter.js +351 -0
- package/dist/pipelines-iceberg-emitter.js.map +1 -0
- package/dist/provider-capabilities.d.ts +146 -0
- package/dist/provider-capabilities.d.ts.map +1 -0
- package/dist/provider-capabilities.js +214 -0
- package/dist/provider-capabilities.js.map +1 -0
- package/dist/rdb-provider-adapter.d.ts +195 -0
- package/dist/rdb-provider-adapter.d.ts.map +1 -0
- package/dist/rdb-provider-adapter.js +291 -0
- package/dist/rdb-provider-adapter.js.map +1 -0
- package/dist/schema/cascade.d.ts +48 -17
- package/dist/schema/cascade.d.ts.map +1 -1
- package/dist/schema/cascade.js +477 -278
- package/dist/schema/cascade.js.map +1 -1
- package/dist/schema/definition-caches.d.ts +24 -0
- package/dist/schema/definition-caches.d.ts.map +1 -0
- package/dist/schema/definition-caches.js +26 -0
- package/dist/schema/definition-caches.js.map +1 -0
- package/dist/schema/dependency-graph.d.ts +21 -109
- package/dist/schema/dependency-graph.d.ts.map +1 -1
- package/dist/schema/dependency-graph.js +25 -333
- package/dist/schema/dependency-graph.js.map +1 -1
- package/dist/schema/diff.d.ts +103 -0
- package/dist/schema/diff.d.ts.map +1 -0
- package/dist/schema/diff.js +329 -0
- package/dist/schema/diff.js.map +1 -0
- package/dist/schema/entity-operations.d.ts +99 -0
- package/dist/schema/entity-operations.d.ts.map +1 -0
- package/dist/schema/entity-operations.js +818 -0
- package/dist/schema/entity-operations.js.map +1 -0
- package/dist/schema/index.d.ts +28 -34
- package/dist/schema/index.d.ts.map +1 -1
- package/dist/schema/index.js +454 -521
- package/dist/schema/index.js.map +1 -1
- package/dist/schema/migration.d.ts +205 -0
- package/dist/schema/migration.d.ts.map +1 -0
- package/dist/schema/migration.js +327 -0
- package/dist/schema/migration.js.map +1 -0
- package/dist/schema/nl-query-generator.d.ts +68 -0
- package/dist/schema/nl-query-generator.d.ts.map +1 -0
- package/dist/schema/nl-query-generator.js +362 -0
- package/dist/schema/nl-query-generator.js.map +1 -0
- package/dist/schema/nl-query.d.ts +65 -0
- package/dist/schema/nl-query.d.ts.map +1 -0
- package/dist/schema/nl-query.js +178 -0
- package/dist/schema/nl-query.js.map +1 -0
- package/dist/schema/parse.d.ts.map +1 -1
- package/dist/schema/parse.js +144 -89
- package/dist/schema/parse.js.map +1 -1
- package/dist/schema/provider.d.ts +37 -0
- package/dist/schema/provider.d.ts.map +1 -1
- package/dist/schema/provider.js +15 -7
- package/dist/schema/provider.js.map +1 -1
- package/dist/schema/resolve.d.ts +46 -5
- package/dist/schema/resolve.d.ts.map +1 -1
- package/dist/schema/resolve.js +237 -95
- package/dist/schema/resolve.js.map +1 -1
- package/dist/schema/search-utils.d.ts +76 -0
- package/dist/schema/search-utils.d.ts.map +1 -0
- package/dist/schema/search-utils.js +86 -0
- package/dist/schema/search-utils.js.map +1 -0
- package/dist/schema/seed.d.ts +53 -0
- package/dist/schema/seed.d.ts.map +1 -0
- package/dist/schema/seed.js +94 -0
- package/dist/schema/seed.js.map +1 -0
- package/dist/schema/semantic.d.ts +10 -0
- package/dist/schema/semantic.d.ts.map +1 -1
- package/dist/schema/semantic.js +192 -86
- package/dist/schema/semantic.js.map +1 -1
- package/dist/schema/sub-apis.d.ts +52 -0
- package/dist/schema/sub-apis.d.ts.map +1 -0
- package/dist/schema/sub-apis.js +216 -0
- package/dist/schema/sub-apis.js.map +1 -0
- package/dist/schema/system-entities.d.ts +42 -0
- package/dist/schema/system-entities.d.ts.map +1 -0
- package/dist/schema/system-entities.js +101 -0
- package/dist/schema/system-entities.js.map +1 -0
- package/dist/schema/types.d.ts +91 -9
- package/dist/schema/types.d.ts.map +1 -1
- package/dist/schema/union-fallback.d.ts.map +1 -1
- package/dist/schema/union-fallback.js +21 -15
- package/dist/schema/union-fallback.js.map +1 -1
- package/dist/schema/value-generators/ai.d.ts +54 -0
- package/dist/schema/value-generators/ai.d.ts.map +1 -0
- package/dist/schema/value-generators/ai.js +136 -0
- package/dist/schema/value-generators/ai.js.map +1 -0
- package/dist/schema/value-generators/index.d.ts +126 -0
- package/dist/schema/value-generators/index.d.ts.map +1 -0
- package/dist/schema/value-generators/index.js +219 -0
- package/dist/schema/value-generators/index.js.map +1 -0
- package/dist/schema/value-generators/placeholder.d.ts +52 -0
- package/dist/schema/value-generators/placeholder.d.ts.map +1 -0
- package/dist/schema/value-generators/placeholder.js +328 -0
- package/dist/schema/value-generators/placeholder.js.map +1 -0
- package/dist/schema/value-generators/types.d.ts +116 -0
- package/dist/schema/value-generators/types.d.ts.map +1 -0
- package/dist/schema/value-generators/types.js +11 -0
- package/dist/schema/value-generators/types.js.map +1 -0
- package/dist/schema/version.d.ts +111 -0
- package/dist/schema/version.d.ts.map +1 -0
- package/dist/schema/version.js +190 -0
- package/dist/schema/version.js.map +1 -0
- package/dist/schema.d.ts +1095 -24
- package/dist/schema.d.ts.map +1 -1
- package/dist/schema.js +2852 -40
- package/dist/schema.js.map +1 -1
- package/dist/semantic-vectors.d.ts +39 -0
- package/dist/semantic-vectors.d.ts.map +1 -0
- package/dist/semantic-vectors.js +334 -0
- package/dist/semantic-vectors.js.map +1 -0
- package/dist/semantic.d.ts +29 -1
- package/dist/semantic.d.ts.map +1 -1
- package/dist/semantic.js +26 -16
- package/dist/semantic.js.map +1 -1
- package/dist/telemetry.d.ts +128 -0
- package/dist/telemetry.d.ts.map +1 -0
- package/dist/telemetry.js +305 -0
- package/dist/telemetry.js.map +1 -0
- package/dist/tests.d.ts.map +1 -1
- package/dist/tests.js +30 -22
- package/dist/tests.js.map +1 -1
- package/dist/type-guards.d.ts +50 -5
- package/dist/type-guards.d.ts.map +1 -1
- package/dist/type-guards.js +87 -16
- package/dist/type-guards.js.map +1 -1
- package/dist/types.d.ts +33 -245
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js +62 -72
- package/dist/types.js.map +1 -1
- package/dist/validation.d.ts +2 -5
- package/dist/validation.d.ts.map +1 -1
- package/dist/validation.js +65 -93
- package/dist/validation.js.map +1 -1
- package/dist/worker/db-provider.d.ts +168 -0
- package/dist/worker/db-provider.d.ts.map +1 -0
- package/dist/worker/db-provider.js +277 -0
- package/dist/worker/db-provider.js.map +1 -0
- package/dist/worker/index.d.ts +35 -0
- package/dist/worker/index.d.ts.map +1 -0
- package/dist/worker/index.js +37 -0
- package/dist/worker/index.js.map +1 -0
- package/dist/worker.d.ts +779 -0
- package/dist/worker.d.ts.map +1 -0
- package/dist/worker.js +2786 -0
- package/dist/worker.js.map +1 -0
- package/package.json +46 -16
- package/src/docs-rels/migrations/0001-init.sql +125 -0
- package/LICENSE +0 -21
|
@@ -0,0 +1,929 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ClickHouse DBProvider adapter (Stack A analytics layer)
|
|
3
|
+
*
|
|
4
|
+
* First-class adapter for the analytical leg of Stack A per
|
|
5
|
+
* [ADR-0003](../../../docs/adr/0003-storage-strategy-pg-clickhouse-default.md).
|
|
6
|
+
* Implements the {@link DBProviderPort} surface with:
|
|
7
|
+
*
|
|
8
|
+
* - **Tier 1 (entity CRUD)** against a `things` table backed by a
|
|
9
|
+
* `ReplacingMergeTree` so updates produce a logical upsert at merge time.
|
|
10
|
+
* - **Tier 2 (graph traversal)** via Actions: subject/object/recipient/...
|
|
11
|
+
* queries against an append-only `actions` table.
|
|
12
|
+
* - **Tier 3 (analytics)** declared **first-class** — ClickHouse is the
|
|
13
|
+
* substrate for cross-cascade aggregations, time-series rollups, and
|
|
14
|
+
* large-scan analytical queries (cf. ADR-0003 "Tier 3 first-class on
|
|
15
|
+
* ClickHouse"). `analyticsQuery` is exposed for ad-hoc SQL.
|
|
16
|
+
* - **Tier 4 (vector search)** native via ClickHouse's vector distance
|
|
17
|
+
* functions (`cosineDistance`, `L2Distance`, `dotProduct`) up to
|
|
18
|
+
* ~64,000 dimensions. Embeddings live in a companion `embeddings`
|
|
19
|
+
* table (`ns, thing_id, type, embedding Array(Float32)`) joined to
|
|
20
|
+
* `things`. Callers seed it via {@link ClickHouseProvider.upsertEmbedding}
|
|
21
|
+
* and query via {@link ClickHouseProvider.vectorSearch}. Frame-aware
|
|
22
|
+
* role filtering is deferred to a follow-up refinement bead.
|
|
23
|
+
* - **SVO Action recording** + **Verb registry** declared via
|
|
24
|
+
* `hasActionRecording: true` and `hasVerbRegistry: true`.
|
|
25
|
+
* - **Sharding**: `unsharded` by default (ClickHouse's strength is wide
|
|
26
|
+
* tables on a single cluster). The `ns` column doubles as a partition
|
|
27
|
+
* key for callers that want logical tenant separation; declaring
|
|
28
|
+
* `partitioned-by-tenant` switches the capability flag.
|
|
29
|
+
*
|
|
30
|
+
* ## Cascade write path
|
|
31
|
+
*
|
|
32
|
+
* The cascade-scale write path on ClickHouse is **bulk INSERT via
|
|
33
|
+
* `JSONEachRow` format** — a single HTTP POST per batch carries the full
|
|
34
|
+
* write set. {@link ClickHouseProvider.commitBatch} implements this shape;
|
|
35
|
+
* each batch becomes one HTTP request.
|
|
36
|
+
*
|
|
37
|
+
* ## Driver choice
|
|
38
|
+
*
|
|
39
|
+
* The adapter is HTTP-based and driver-agnostic. It consumes a
|
|
40
|
+
* {@link ClickHouseHttpFetcher} — a function that takes a SQL query plus
|
|
41
|
+
* an optional body and returns the response text. Any HTTP client
|
|
42
|
+
* (`fetch`, `node-fetch`, `undici`, Cloudflare Workers `fetch`) works.
|
|
43
|
+
*
|
|
44
|
+
* For ad-hoc scripting against a ClickHouse Cloud or self-hosted instance,
|
|
45
|
+
* {@link createClickHouseHttpFetcher} wraps a basic-auth fetch call.
|
|
46
|
+
*
|
|
47
|
+
* ## Schema
|
|
48
|
+
*
|
|
49
|
+
* The adapter assumes a database named `aidb` (configurable). DDL:
|
|
50
|
+
*
|
|
51
|
+
* ```sql
|
|
52
|
+
* CREATE DATABASE IF NOT EXISTS aidb;
|
|
53
|
+
*
|
|
54
|
+
* CREATE TABLE IF NOT EXISTS aidb.things (
|
|
55
|
+
* ns String,
|
|
56
|
+
* id String,
|
|
57
|
+
* type String,
|
|
58
|
+
* data String, -- JSON-encoded payload
|
|
59
|
+
* created_at DateTime64(3) DEFAULT now64(3),
|
|
60
|
+
* updated_at DateTime64(3) DEFAULT now64(3),
|
|
61
|
+
* version UInt64 DEFAULT toUnixTimestamp64Milli(now64(3))
|
|
62
|
+
* ) ENGINE = ReplacingMergeTree(version)
|
|
63
|
+
* ORDER BY (ns, type, id);
|
|
64
|
+
*
|
|
65
|
+
* CREATE TABLE IF NOT EXISTS aidb.actions (
|
|
66
|
+
* ns String,
|
|
67
|
+
* id String,
|
|
68
|
+
* verb String,
|
|
69
|
+
* subject String,
|
|
70
|
+
* object String,
|
|
71
|
+
* roles String, -- JSON-encoded role map
|
|
72
|
+
* data String, -- JSON-encoded payload
|
|
73
|
+
* status LowCardinality(String) DEFAULT 'pending',
|
|
74
|
+
* created_at DateTime64(3) DEFAULT now64(3),
|
|
75
|
+
* completed_at Nullable(DateTime64(3))
|
|
76
|
+
* ) ENGINE = MergeTree
|
|
77
|
+
* PARTITION BY toYYYYMM(created_at)
|
|
78
|
+
* ORDER BY (ns, verb, subject, created_at);
|
|
79
|
+
*
|
|
80
|
+
* CREATE TABLE IF NOT EXISTS aidb.verbs (
|
|
81
|
+
* name String,
|
|
82
|
+
* data String,
|
|
83
|
+
* created_at DateTime64(3) DEFAULT now64(3),
|
|
84
|
+
* version UInt64 DEFAULT toUnixTimestamp64Milli(now64(3))
|
|
85
|
+
* ) ENGINE = ReplacingMergeTree(version)
|
|
86
|
+
* ORDER BY name;
|
|
87
|
+
* ```
|
|
88
|
+
*
|
|
89
|
+
* `bootstrapClickHouseSchema(fetcher)` ships the DDL above.
|
|
90
|
+
*
|
|
91
|
+
* @packageDocumentation
|
|
92
|
+
*/
|
|
93
|
+
import { validateTypeName, validateEntityId, validateEntityData, validateRelationName, validateSearchQuery, validateListOptions, validateSearchOptions, } from './validation.js';
|
|
94
|
+
import { EntityNotFoundError } from './errors.js';
|
|
95
|
+
/**
|
|
96
|
+
* Wrap a `fetch`-compatible function into a {@link ClickHouseHttpFetcher}.
|
|
97
|
+
*
|
|
98
|
+
* Sends queries via POST with the SQL as the request body for SELECTs and
|
|
99
|
+
* with the data block as the body when `body` is supplied (insert path).
|
|
100
|
+
* Basic-auth credentials are encoded into the `Authorization` header.
|
|
101
|
+
*
|
|
102
|
+
* @example
|
|
103
|
+
* ```ts
|
|
104
|
+
* import { createClickHouseHttpFetcher, createClickHouseProvider } from 'ai-database'
|
|
105
|
+
*
|
|
106
|
+
* const fetcher = createClickHouseHttpFetcher({
|
|
107
|
+
* url: env.CLICKHOUSE_URL,
|
|
108
|
+
* username: env.CLICKHOUSE_USER,
|
|
109
|
+
* password: env.CLICKHOUSE_PASSWORD,
|
|
110
|
+
* database: 'aidb',
|
|
111
|
+
* })
|
|
112
|
+
* const provider = createClickHouseProvider({
|
|
113
|
+
* fetcher,
|
|
114
|
+
* namespace: 'tenant-9',
|
|
115
|
+
* })
|
|
116
|
+
* ```
|
|
117
|
+
*/
|
|
118
|
+
export function createClickHouseHttpFetcher(options) {
|
|
119
|
+
const fetchImpl = options.fetchImpl ?? fetch;
|
|
120
|
+
const auth = options.username !== undefined
|
|
121
|
+
? 'Basic ' + btoa(`${options.username}:${options.password ?? ''}`)
|
|
122
|
+
: undefined;
|
|
123
|
+
const dbParam = options.database ? `database=${encodeURIComponent(options.database)}` : null;
|
|
124
|
+
return async (query, body) => {
|
|
125
|
+
const params = [];
|
|
126
|
+
if (dbParam)
|
|
127
|
+
params.push(dbParam);
|
|
128
|
+
// When body is supplied (insert path), the SQL goes via the `query=` param
|
|
129
|
+
// and the body carries the rows. Otherwise, send SQL as the body.
|
|
130
|
+
let url = options.url;
|
|
131
|
+
let requestBody;
|
|
132
|
+
if (body !== undefined) {
|
|
133
|
+
params.push(`query=${encodeURIComponent(query)}`);
|
|
134
|
+
requestBody = body;
|
|
135
|
+
}
|
|
136
|
+
else {
|
|
137
|
+
requestBody = query;
|
|
138
|
+
}
|
|
139
|
+
if (params.length > 0)
|
|
140
|
+
url = `${url}?${params.join('&')}`;
|
|
141
|
+
const headers = {
|
|
142
|
+
'Content-Type': 'text/plain; charset=utf-8',
|
|
143
|
+
};
|
|
144
|
+
if (auth)
|
|
145
|
+
headers['Authorization'] = auth;
|
|
146
|
+
const response = await fetchImpl(url, {
|
|
147
|
+
method: 'POST',
|
|
148
|
+
headers,
|
|
149
|
+
body: requestBody,
|
|
150
|
+
});
|
|
151
|
+
const text = await response.text();
|
|
152
|
+
if (!response.ok) {
|
|
153
|
+
throw new Error(`ClickHouse HTTP ${response.status}: ${text.slice(0, 500)}`);
|
|
154
|
+
}
|
|
155
|
+
return text;
|
|
156
|
+
};
|
|
157
|
+
}
|
|
158
|
+
// =============================================================================
|
|
159
|
+
// SQL helpers
|
|
160
|
+
// =============================================================================
|
|
161
|
+
const DEFAULT_DATABASE = 'aidb';
|
|
162
|
+
const DEFAULT_NAMESPACE = 'default';
|
|
163
|
+
const DEFAULT_VECTOR_DIMS = 1536;
|
|
164
|
+
const CH_MAX_VECTOR_DIMS = 64_000;
|
|
165
|
+
/** Generate a UUID for adapter-issued ids. */
|
|
166
|
+
function genId() {
|
|
167
|
+
return crypto.randomUUID();
|
|
168
|
+
}
|
|
169
|
+
/**
|
|
170
|
+
* SQL string-literal escape — ClickHouse accepts single-quoted strings
|
|
171
|
+
* with backslash escapes.
|
|
172
|
+
*/
|
|
173
|
+
function quote(value) {
|
|
174
|
+
if (value === null || value === undefined)
|
|
175
|
+
return 'NULL';
|
|
176
|
+
return `'${value.replace(/\\/g, '\\\\').replace(/'/g, "\\'")}'`;
|
|
177
|
+
}
|
|
178
|
+
/**
|
|
179
|
+
* Coerce arbitrary string-encoded JSON column values into a plain object.
|
|
180
|
+
* The schema stores `data`, `roles` as String for portability across
|
|
181
|
+
* ClickHouse versions — JSON type support varies by deployment.
|
|
182
|
+
*/
|
|
183
|
+
function asJsonb(value) {
|
|
184
|
+
if (!value)
|
|
185
|
+
return {};
|
|
186
|
+
if (typeof value === 'string') {
|
|
187
|
+
if (value.length === 0)
|
|
188
|
+
return {};
|
|
189
|
+
try {
|
|
190
|
+
return JSON.parse(value);
|
|
191
|
+
}
|
|
192
|
+
catch {
|
|
193
|
+
return {};
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
if (typeof value === 'object')
|
|
197
|
+
return value;
|
|
198
|
+
return {};
|
|
199
|
+
}
|
|
200
|
+
function asDate(value) {
|
|
201
|
+
if (!value)
|
|
202
|
+
return undefined;
|
|
203
|
+
if (value instanceof Date)
|
|
204
|
+
return value;
|
|
205
|
+
if (typeof value === 'string' || typeof value === 'number') {
|
|
206
|
+
const d = new Date(value);
|
|
207
|
+
return Number.isNaN(d.getTime()) ? undefined : d;
|
|
208
|
+
}
|
|
209
|
+
return undefined;
|
|
210
|
+
}
|
|
211
|
+
function asString(value) {
|
|
212
|
+
if (value === null || value === undefined)
|
|
213
|
+
return undefined;
|
|
214
|
+
if (typeof value === 'string' && value.length === 0)
|
|
215
|
+
return undefined;
|
|
216
|
+
return String(value);
|
|
217
|
+
}
|
|
218
|
+
/**
|
|
219
|
+
* Format a Date as ClickHouse `DateTime64(3)`-compatible literal:
|
|
220
|
+
* `'2026-05-05 12:34:56.789'`.
|
|
221
|
+
*/
|
|
222
|
+
function chDateTime(date) {
|
|
223
|
+
// ClickHouse accepts `YYYY-MM-DD HH:MM:SS.sss` in single-quotes.
|
|
224
|
+
const iso = date.toISOString();
|
|
225
|
+
// 2026-05-05T12:34:56.789Z -> 2026-05-05 12:34:56.789
|
|
226
|
+
return iso.replace('T', ' ').replace('Z', '');
|
|
227
|
+
}
|
|
228
|
+
function parseJsonResponse(text) {
|
|
229
|
+
if (!text || text.length === 0)
|
|
230
|
+
return [];
|
|
231
|
+
try {
|
|
232
|
+
const parsed = JSON.parse(text);
|
|
233
|
+
return parsed.data ?? [];
|
|
234
|
+
}
|
|
235
|
+
catch {
|
|
236
|
+
// Some ClickHouse responses for non-SELECT statements are empty or
|
|
237
|
+
// plain text; treat as no data.
|
|
238
|
+
return [];
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
// =============================================================================
|
|
242
|
+
// ClickHouseProvider
|
|
243
|
+
// =============================================================================
|
|
244
|
+
/**
|
|
245
|
+
* ClickHouse adapter implementing {@link DBProviderPort} and
|
|
246
|
+
* {@link DBProviderSVO}.
|
|
247
|
+
*/
|
|
248
|
+
export class ClickHouseProvider {
|
|
249
|
+
fetcher;
|
|
250
|
+
namespace;
|
|
251
|
+
database;
|
|
252
|
+
_shardingModel;
|
|
253
|
+
vectorDimensions;
|
|
254
|
+
driver;
|
|
255
|
+
constructor(options) {
|
|
256
|
+
this.fetcher = options.fetcher;
|
|
257
|
+
this.namespace = options.namespace ?? DEFAULT_NAMESPACE;
|
|
258
|
+
this.database = options.database ?? DEFAULT_DATABASE;
|
|
259
|
+
this._shardingModel = options.shardingModel ?? 'unsharded';
|
|
260
|
+
this.vectorDimensions = Math.min(options.vectorDimensions ?? DEFAULT_VECTOR_DIMS, CH_MAX_VECTOR_DIMS);
|
|
261
|
+
this.driver = options.driver ?? 'clickhouse-http';
|
|
262
|
+
}
|
|
263
|
+
/**
|
|
264
|
+
* Tier capability declaration. Declares Tier 3 first-class (CH's
|
|
265
|
+
* strength) and Tier 4 (native vector functions, up to 64,000 dims,
|
|
266
|
+
* cosine/L2/dot). Both `hasActionRecording` and `hasVerbRegistry` are
|
|
267
|
+
* `true` — see the SVO methods on this class.
|
|
268
|
+
*/
|
|
269
|
+
get capabilities() {
|
|
270
|
+
return {
|
|
271
|
+
adapter: 'clickhouse',
|
|
272
|
+
shardingModel: this._shardingModel,
|
|
273
|
+
analytics: {
|
|
274
|
+
hasAggregations: true,
|
|
275
|
+
hasTimeSeries: true,
|
|
276
|
+
hasLargeScans: true,
|
|
277
|
+
},
|
|
278
|
+
vectorSearch: {
|
|
279
|
+
maxDimensions: this.vectorDimensions,
|
|
280
|
+
metrics: ['cosine', 'l2', 'dot'],
|
|
281
|
+
implementation: 'native',
|
|
282
|
+
},
|
|
283
|
+
hasActionRecording: true,
|
|
284
|
+
hasVerbRegistry: true,
|
|
285
|
+
};
|
|
286
|
+
}
|
|
287
|
+
// ===========================================================================
|
|
288
|
+
// Tier 1 — Entity CRUD
|
|
289
|
+
// ===========================================================================
|
|
290
|
+
async get(type, id) {
|
|
291
|
+
validateTypeName(type);
|
|
292
|
+
validateEntityId(id);
|
|
293
|
+
const sql = `SELECT data FROM ${this.database}.things FINAL
|
|
294
|
+
WHERE ns = ${quote(this.namespace)} AND type = ${quote(type)} AND id = ${quote(id)}
|
|
295
|
+
LIMIT 1 FORMAT JSON`;
|
|
296
|
+
const text = await this.fetcher(sql);
|
|
297
|
+
const rows = parseJsonResponse(text);
|
|
298
|
+
if (rows.length === 0)
|
|
299
|
+
return null;
|
|
300
|
+
const data = asJsonb(rows[0]['data']);
|
|
301
|
+
return { ...data, $id: id, $type: type };
|
|
302
|
+
}
|
|
303
|
+
async list(type, options) {
|
|
304
|
+
validateTypeName(type);
|
|
305
|
+
validateListOptions(options);
|
|
306
|
+
const limit = options?.limit ?? 1000;
|
|
307
|
+
const offset = options?.offset ?? 0;
|
|
308
|
+
const sql = `SELECT id, data FROM ${this.database}.things FINAL
|
|
309
|
+
WHERE ns = ${quote(this.namespace)} AND type = ${quote(type)}
|
|
310
|
+
ORDER BY id
|
|
311
|
+
LIMIT ${Number(limit)} OFFSET ${Number(offset)}
|
|
312
|
+
FORMAT JSON`;
|
|
313
|
+
const text = await this.fetcher(sql);
|
|
314
|
+
const rows = parseJsonResponse(text);
|
|
315
|
+
let result = rows.map((row) => {
|
|
316
|
+
const data = asJsonb(row['data']);
|
|
317
|
+
return { ...data, $id: String(row['id']), $type: type };
|
|
318
|
+
});
|
|
319
|
+
// Apply where filter client-side for parity with MemoryProvider.
|
|
320
|
+
// For high-volume paths, callers should use raw SQL via
|
|
321
|
+
// analyticsQuery() — this is the simple convenience path.
|
|
322
|
+
if (options?.where) {
|
|
323
|
+
result = result.filter((entity) => {
|
|
324
|
+
for (const [key, value] of Object.entries(options.where)) {
|
|
325
|
+
if (entity[key] !== value)
|
|
326
|
+
return false;
|
|
327
|
+
}
|
|
328
|
+
return true;
|
|
329
|
+
});
|
|
330
|
+
}
|
|
331
|
+
return result;
|
|
332
|
+
}
|
|
333
|
+
async search(type, query, options) {
|
|
334
|
+
validateTypeName(type);
|
|
335
|
+
validateSearchQuery(query);
|
|
336
|
+
validateSearchOptions(options);
|
|
337
|
+
const limit = options?.limit ?? 100;
|
|
338
|
+
// ClickHouse `like` against the JSON-string column. Richer search
|
|
339
|
+
// (FTS via skipping indexes) is future work.
|
|
340
|
+
const pattern = `%${query.replace(/\\/g, '\\\\').replace(/'/g, "\\'")}%`;
|
|
341
|
+
const sql = `SELECT id, data FROM ${this.database}.things FINAL
|
|
342
|
+
WHERE ns = ${quote(this.namespace)} AND type = ${quote(type)} AND data ILIKE '${pattern}'
|
|
343
|
+
ORDER BY id
|
|
344
|
+
LIMIT ${Number(limit)}
|
|
345
|
+
FORMAT JSON`;
|
|
346
|
+
const text = await this.fetcher(sql);
|
|
347
|
+
const rows = parseJsonResponse(text);
|
|
348
|
+
return rows.map((row) => {
|
|
349
|
+
const data = asJsonb(row['data']);
|
|
350
|
+
return { ...data, $id: String(row['id']), $type: type };
|
|
351
|
+
});
|
|
352
|
+
}
|
|
353
|
+
async create(type, id, data) {
|
|
354
|
+
validateTypeName(type);
|
|
355
|
+
if (id !== undefined)
|
|
356
|
+
validateEntityId(id);
|
|
357
|
+
validateEntityData(data);
|
|
358
|
+
const entityId = id ?? genId();
|
|
359
|
+
const now = new Date();
|
|
360
|
+
const nowIso = now.toISOString();
|
|
361
|
+
const payload = { ...data, createdAt: nowIso, updatedAt: nowIso };
|
|
362
|
+
// INSERT via JSONEachRow body
|
|
363
|
+
const row = {
|
|
364
|
+
ns: this.namespace,
|
|
365
|
+
id: entityId,
|
|
366
|
+
type,
|
|
367
|
+
data: JSON.stringify(payload),
|
|
368
|
+
created_at: chDateTime(now),
|
|
369
|
+
updated_at: chDateTime(now),
|
|
370
|
+
version: now.getTime(),
|
|
371
|
+
};
|
|
372
|
+
await this.fetcher(`INSERT INTO ${this.database}.things (ns, id, type, data, created_at, updated_at, version) FORMAT JSONEachRow`, JSON.stringify(row) + '\n');
|
|
373
|
+
return { ...payload, $id: entityId, $type: type };
|
|
374
|
+
}
|
|
375
|
+
async update(type, id, data) {
|
|
376
|
+
validateTypeName(type);
|
|
377
|
+
validateEntityId(id);
|
|
378
|
+
validateEntityData(data);
|
|
379
|
+
const existing = await this.get(type, id);
|
|
380
|
+
if (!existing)
|
|
381
|
+
throw new EntityNotFoundError(type, id, 'update');
|
|
382
|
+
const { $id: _id, $type: _type, ...rest } = existing;
|
|
383
|
+
const now = new Date();
|
|
384
|
+
const merged = { ...rest, ...data, updatedAt: now.toISOString() };
|
|
385
|
+
// ReplacingMergeTree: a fresh row with a higher `version` supersedes
|
|
386
|
+
// the prior row on merge. For read-after-write consistency we use
|
|
387
|
+
// `FINAL` in `get()`/`list()`.
|
|
388
|
+
const row = {
|
|
389
|
+
ns: this.namespace,
|
|
390
|
+
id,
|
|
391
|
+
type,
|
|
392
|
+
data: JSON.stringify(merged),
|
|
393
|
+
created_at: chDateTime(now),
|
|
394
|
+
updated_at: chDateTime(now),
|
|
395
|
+
version: now.getTime(),
|
|
396
|
+
};
|
|
397
|
+
await this.fetcher(`INSERT INTO ${this.database}.things (ns, id, type, data, created_at, updated_at, version) FORMAT JSONEachRow`, JSON.stringify(row) + '\n');
|
|
398
|
+
return { ...merged, $id: id, $type: type };
|
|
399
|
+
}
|
|
400
|
+
async delete(type, id) {
|
|
401
|
+
validateTypeName(type);
|
|
402
|
+
validateEntityId(id);
|
|
403
|
+
const existing = await this.get(type, id);
|
|
404
|
+
if (!existing)
|
|
405
|
+
return false;
|
|
406
|
+
// ALTER ... DELETE is asynchronous on MergeTree; for synchronous
|
|
407
|
+
// visibility we use lightweight DELETE (CH 23.3+) with mutations
|
|
408
|
+
// SETTINGS to wait. Fallback path uses ALTER ... DELETE.
|
|
409
|
+
await this.fetcher(`DELETE FROM ${this.database}.things WHERE ns = ${quote(this.namespace)} AND type = ${quote(type)} AND id = ${quote(id)}`);
|
|
410
|
+
await this.fetcher(`DELETE FROM ${this.database}.actions WHERE ns = ${quote(this.namespace)} AND (subject = ${quote(id)} OR object = ${quote(id)})`);
|
|
411
|
+
return true;
|
|
412
|
+
}
|
|
413
|
+
// ===========================================================================
|
|
414
|
+
// Tier 2 — Graph traversal via Actions
|
|
415
|
+
// ===========================================================================
|
|
416
|
+
async related(type, id, relation) {
|
|
417
|
+
validateTypeName(type);
|
|
418
|
+
validateEntityId(id);
|
|
419
|
+
validateRelationName(relation);
|
|
420
|
+
const sql = `SELECT t.id AS id, t.type AS type, t.data AS data
|
|
421
|
+
FROM ${this.database}.actions a
|
|
422
|
+
INNER JOIN ${this.database}.things FINAL t
|
|
423
|
+
ON t.ns = a.ns AND t.id = a.object
|
|
424
|
+
WHERE a.ns = ${quote(this.namespace)} AND a.subject = ${quote(id)} AND a.verb = ${quote(relation)}
|
|
425
|
+
FORMAT JSON`;
|
|
426
|
+
const text = await this.fetcher(sql);
|
|
427
|
+
const rows = parseJsonResponse(text);
|
|
428
|
+
return rows.map((row) => {
|
|
429
|
+
const data = asJsonb(row['data']);
|
|
430
|
+
return {
|
|
431
|
+
...data,
|
|
432
|
+
$id: String(row['id']),
|
|
433
|
+
$type: String(row['type']),
|
|
434
|
+
};
|
|
435
|
+
});
|
|
436
|
+
}
|
|
437
|
+
async relate(fromType, fromId, relation, toType, toId, metadata) {
|
|
438
|
+
validateTypeName(fromType);
|
|
439
|
+
validateEntityId(fromId);
|
|
440
|
+
validateRelationName(relation);
|
|
441
|
+
validateTypeName(toType);
|
|
442
|
+
validateEntityId(toId);
|
|
443
|
+
await this.recordAction({
|
|
444
|
+
verb: relation,
|
|
445
|
+
subject: fromId,
|
|
446
|
+
object: toId,
|
|
447
|
+
data: {
|
|
448
|
+
fromType,
|
|
449
|
+
toType,
|
|
450
|
+
...(metadata ? { metadata } : {}),
|
|
451
|
+
},
|
|
452
|
+
status: 'completed',
|
|
453
|
+
});
|
|
454
|
+
}
|
|
455
|
+
async unrelate(fromType, fromId, relation, toType, toId) {
|
|
456
|
+
validateTypeName(fromType);
|
|
457
|
+
validateEntityId(fromId);
|
|
458
|
+
validateRelationName(relation);
|
|
459
|
+
validateTypeName(toType);
|
|
460
|
+
validateEntityId(toId);
|
|
461
|
+
await this.fetcher(`DELETE FROM ${this.database}.actions
|
|
462
|
+
WHERE ns = ${quote(this.namespace)} AND verb = ${quote(relation)} AND subject = ${quote(fromId)} AND object = ${quote(toId)}`);
|
|
463
|
+
}
|
|
464
|
+
// ===========================================================================
|
|
465
|
+
// SVO Action recording (DBProviderSVO)
|
|
466
|
+
// ===========================================================================
|
|
467
|
+
async recordAction(input) {
|
|
468
|
+
const id = genId();
|
|
469
|
+
const status = input.status ?? 'pending';
|
|
470
|
+
const createdAt = new Date();
|
|
471
|
+
const completedAt = status === 'completed' || status === 'failed' || status === 'cancelled'
|
|
472
|
+
? createdAt
|
|
473
|
+
: undefined;
|
|
474
|
+
const row = {
|
|
475
|
+
ns: this.namespace,
|
|
476
|
+
id,
|
|
477
|
+
verb: input.verb,
|
|
478
|
+
subject: input.subject ?? '',
|
|
479
|
+
object: input.object ?? '',
|
|
480
|
+
roles: JSON.stringify(input.roles ?? {}),
|
|
481
|
+
data: JSON.stringify(input.data ?? {}),
|
|
482
|
+
status,
|
|
483
|
+
created_at: chDateTime(createdAt),
|
|
484
|
+
completed_at: completedAt ? chDateTime(completedAt) : null,
|
|
485
|
+
};
|
|
486
|
+
await this.fetcher(`INSERT INTO ${this.database}.actions
|
|
487
|
+
(ns, id, verb, subject, object, roles, data, status, created_at, completed_at)
|
|
488
|
+
FORMAT JSONEachRow`, JSON.stringify(row) + '\n');
|
|
489
|
+
const action = {
|
|
490
|
+
id,
|
|
491
|
+
verb: input.verb,
|
|
492
|
+
...(input.subject !== undefined && { subject: input.subject }),
|
|
493
|
+
...(input.object !== undefined && { object: input.object }),
|
|
494
|
+
...(input.roles !== undefined && { roles: input.roles }),
|
|
495
|
+
...(input.data !== undefined && { data: input.data }),
|
|
496
|
+
status,
|
|
497
|
+
createdAt,
|
|
498
|
+
...(completedAt !== undefined && { completedAt }),
|
|
499
|
+
};
|
|
500
|
+
return action;
|
|
501
|
+
}
|
|
502
|
+
async queryActions(query = {}) {
|
|
503
|
+
const conditions = [`ns = ${quote(this.namespace)}`];
|
|
504
|
+
if (query.verb !== undefined) {
|
|
505
|
+
conditions.push(`verb = ${quote(query.verb)}`);
|
|
506
|
+
}
|
|
507
|
+
if (query.subject !== undefined) {
|
|
508
|
+
conditions.push(`subject = ${quote(query.subject)}`);
|
|
509
|
+
}
|
|
510
|
+
if (query.object !== undefined) {
|
|
511
|
+
conditions.push(`object = ${quote(query.object)}`);
|
|
512
|
+
}
|
|
513
|
+
if (query.role) {
|
|
514
|
+
for (const [role, value] of Object.entries(query.role)) {
|
|
515
|
+
if (typeof value !== 'string')
|
|
516
|
+
continue;
|
|
517
|
+
if (role === 'subject' || role === 'object') {
|
|
518
|
+
conditions.push(`${role} = ${quote(value)}`);
|
|
519
|
+
}
|
|
520
|
+
else {
|
|
521
|
+
conditions.push(`JSONExtractString(roles, ${quote(role)}) = ${quote(value)}`);
|
|
522
|
+
}
|
|
523
|
+
}
|
|
524
|
+
}
|
|
525
|
+
if (query.status) {
|
|
526
|
+
const statuses = Array.isArray(query.status) ? query.status : [query.status];
|
|
527
|
+
const inList = statuses.map((s) => quote(s)).join(', ');
|
|
528
|
+
conditions.push(`status IN (${inList})`);
|
|
529
|
+
}
|
|
530
|
+
if (query.since) {
|
|
531
|
+
conditions.push(`created_at >= ${quote(chDateTime(query.since))}`);
|
|
532
|
+
}
|
|
533
|
+
if (query.until) {
|
|
534
|
+
conditions.push(`created_at <= ${quote(chDateTime(query.until))}`);
|
|
535
|
+
}
|
|
536
|
+
const limit = query.limit ?? 1000;
|
|
537
|
+
const offset = query.offset ?? 0;
|
|
538
|
+
const sql = `SELECT id, verb, subject, object, roles, data, status,
|
|
539
|
+
toString(created_at) AS created_at,
|
|
540
|
+
toString(completed_at) AS completed_at
|
|
541
|
+
FROM ${this.database}.actions
|
|
542
|
+
WHERE ${conditions.join(' AND ')}
|
|
543
|
+
ORDER BY created_at ASC
|
|
544
|
+
LIMIT ${Number(limit)} OFFSET ${Number(offset)}
|
|
545
|
+
FORMAT JSON`;
|
|
546
|
+
const text = await this.fetcher(sql);
|
|
547
|
+
const rows = parseJsonResponse(text);
|
|
548
|
+
return rows.map((row) => {
|
|
549
|
+
const action = {
|
|
550
|
+
id: String(row['id']),
|
|
551
|
+
verb: String(row['verb']),
|
|
552
|
+
status: row['status'] ?? 'pending',
|
|
553
|
+
createdAt: asDate(row['created_at']) ?? new Date(0),
|
|
554
|
+
};
|
|
555
|
+
const subject = asString(row['subject']);
|
|
556
|
+
if (subject !== undefined)
|
|
557
|
+
action.subject = subject;
|
|
558
|
+
const object = asString(row['object']);
|
|
559
|
+
if (object !== undefined)
|
|
560
|
+
action.object = object;
|
|
561
|
+
const roles = asJsonb(row['roles']);
|
|
562
|
+
if (Object.keys(roles).length > 0) {
|
|
563
|
+
action.roles = roles;
|
|
564
|
+
}
|
|
565
|
+
const data = asJsonb(row['data']);
|
|
566
|
+
if (Object.keys(data).length > 0) {
|
|
567
|
+
action.data = data;
|
|
568
|
+
}
|
|
569
|
+
const completedAt = asDate(row['completed_at']);
|
|
570
|
+
if (completedAt)
|
|
571
|
+
action.completedAt = completedAt;
|
|
572
|
+
return action;
|
|
573
|
+
});
|
|
574
|
+
}
|
|
575
|
+
// ===========================================================================
|
|
576
|
+
// Verb registry (DBProviderSVO)
|
|
577
|
+
// ===========================================================================
|
|
578
|
+
async defineVerb(def) {
|
|
579
|
+
const verb = {
|
|
580
|
+
name: def.name,
|
|
581
|
+
action: def.action ?? def.name,
|
|
582
|
+
act: def.act ?? `${def.name}s`,
|
|
583
|
+
activity: def.activity ?? `${def.name}ing`,
|
|
584
|
+
event: def.event ?? `${def.name}d`,
|
|
585
|
+
...(def.reverseBy !== undefined && { reverseBy: def.reverseBy }),
|
|
586
|
+
...(def.reverseAt !== undefined && { reverseAt: def.reverseAt }),
|
|
587
|
+
...(def.reverseIn !== undefined && { reverseIn: def.reverseIn }),
|
|
588
|
+
...(def.inverse !== undefined && { inverse: def.inverse }),
|
|
589
|
+
...(def.description !== undefined && { description: def.description }),
|
|
590
|
+
...(def.frame !== undefined && { frame: def.frame }),
|
|
591
|
+
...(def.source !== undefined && { source: def.source }),
|
|
592
|
+
...(def.canonical !== undefined && { canonical: def.canonical }),
|
|
593
|
+
createdAt: new Date(),
|
|
594
|
+
};
|
|
595
|
+
const now = new Date();
|
|
596
|
+
const row = {
|
|
597
|
+
name: verb.name,
|
|
598
|
+
data: JSON.stringify(verb),
|
|
599
|
+
created_at: chDateTime(now),
|
|
600
|
+
version: now.getTime(),
|
|
601
|
+
};
|
|
602
|
+
await this.fetcher(`INSERT INTO ${this.database}.verbs (name, data, created_at, version) FORMAT JSONEachRow`, JSON.stringify(row) + '\n');
|
|
603
|
+
return verb;
|
|
604
|
+
}
|
|
605
|
+
async getVerb(name) {
|
|
606
|
+
const sql = `SELECT data, toString(created_at) AS created_at FROM ${this.database}.verbs FINAL WHERE name = ${quote(name)} LIMIT 1 FORMAT JSON`;
|
|
607
|
+
const text = await this.fetcher(sql);
|
|
608
|
+
const rows = parseJsonResponse(text);
|
|
609
|
+
if (rows.length === 0)
|
|
610
|
+
return null;
|
|
611
|
+
const data = asJsonb(rows[0]['data']);
|
|
612
|
+
const createdAt = asDate(rows[0]['created_at']) ?? new Date(0);
|
|
613
|
+
return { ...data, createdAt };
|
|
614
|
+
}
|
|
615
|
+
async listVerbs() {
|
|
616
|
+
const sql = `SELECT data, toString(created_at) AS created_at FROM ${this.database}.verbs FINAL ORDER BY name ASC FORMAT JSON`;
|
|
617
|
+
const text = await this.fetcher(sql);
|
|
618
|
+
const rows = parseJsonResponse(text);
|
|
619
|
+
return rows.map((row) => {
|
|
620
|
+
const data = asJsonb(row['data']);
|
|
621
|
+
const createdAt = asDate(row['created_at']) ?? new Date(0);
|
|
622
|
+
return { ...data, createdAt };
|
|
623
|
+
});
|
|
624
|
+
}
|
|
625
|
+
// ===========================================================================
|
|
626
|
+
// Cascade write fast path — bulk JSONEachRow
|
|
627
|
+
// ===========================================================================
|
|
628
|
+
/**
|
|
629
|
+
* Bulk-commit Things and Actions in two single-table inserts via
|
|
630
|
+
* `JSONEachRow`. Each table's payload is one HTTP POST, two HTTP
|
|
631
|
+
* requests total per batch. ClickHouse swallows duplicate inserts via
|
|
632
|
+
* the `ReplacingMergeTree` engine on `things` (versioned dedup at merge
|
|
633
|
+
* time) and via append-only writes on `actions` (caller is responsible
|
|
634
|
+
* for deduping ids if needed).
|
|
635
|
+
*
|
|
636
|
+
* Returns the count of rows submitted (CH does not report inserted-row
|
|
637
|
+
* counts via the HTTP interface in a uniform way; the values here
|
|
638
|
+
* reflect the input batch sizes).
|
|
639
|
+
*
|
|
640
|
+
* @example
|
|
641
|
+
* ```ts
|
|
642
|
+
* const { thingsInserted, actionsInserted } = await provider.commitBatch({
|
|
643
|
+
* things: [
|
|
644
|
+
* { type: 'Customer', id: 'c1', data: { name: 'Acme' } },
|
|
645
|
+
* { type: 'Order', id: 'o1', data: { total: 100 } },
|
|
646
|
+
* ],
|
|
647
|
+
* actions: [
|
|
648
|
+
* { id: 'a1', verb: 'placedBy', subject: 'o1', object: 'c1', status: 'completed' },
|
|
649
|
+
* ],
|
|
650
|
+
* })
|
|
651
|
+
* ```
|
|
652
|
+
*/
|
|
653
|
+
async commitBatch(input) {
|
|
654
|
+
const things = input.things ?? [];
|
|
655
|
+
const actions = input.actions ?? [];
|
|
656
|
+
if (things.length === 0 && actions.length === 0) {
|
|
657
|
+
return { thingsInserted: 0, actionsInserted: 0 };
|
|
658
|
+
}
|
|
659
|
+
const now = new Date();
|
|
660
|
+
const nowStr = chDateTime(now);
|
|
661
|
+
const version = now.getTime();
|
|
662
|
+
if (things.length > 0) {
|
|
663
|
+
const lines = things
|
|
664
|
+
.map((t) => JSON.stringify({
|
|
665
|
+
ns: this.namespace,
|
|
666
|
+
id: t.id,
|
|
667
|
+
type: t.type,
|
|
668
|
+
data: JSON.stringify(t.data),
|
|
669
|
+
created_at: nowStr,
|
|
670
|
+
updated_at: nowStr,
|
|
671
|
+
version,
|
|
672
|
+
}))
|
|
673
|
+
.join('\n');
|
|
674
|
+
await this.fetcher(`INSERT INTO ${this.database}.things (ns, id, type, data, created_at, updated_at, version) FORMAT JSONEachRow`, lines + '\n');
|
|
675
|
+
}
|
|
676
|
+
if (actions.length > 0) {
|
|
677
|
+
const lines = actions
|
|
678
|
+
.map((a) => {
|
|
679
|
+
const status = a.status ?? 'pending';
|
|
680
|
+
const completed = status === 'completed' || status === 'failed' || status === 'cancelled' ? nowStr : null;
|
|
681
|
+
return JSON.stringify({
|
|
682
|
+
ns: this.namespace,
|
|
683
|
+
id: a.id ?? genId(),
|
|
684
|
+
verb: a.verb,
|
|
685
|
+
subject: a.subject ?? '',
|
|
686
|
+
object: a.object ?? '',
|
|
687
|
+
roles: JSON.stringify(a.roles ?? {}),
|
|
688
|
+
data: JSON.stringify(a.data ?? {}),
|
|
689
|
+
status,
|
|
690
|
+
created_at: nowStr,
|
|
691
|
+
completed_at: completed,
|
|
692
|
+
});
|
|
693
|
+
})
|
|
694
|
+
.join('\n');
|
|
695
|
+
await this.fetcher(`INSERT INTO ${this.database}.actions
|
|
696
|
+
(ns, id, verb, subject, object, roles, data, status, created_at, completed_at)
|
|
697
|
+
FORMAT JSONEachRow`, lines + '\n');
|
|
698
|
+
}
|
|
699
|
+
return { thingsInserted: things.length, actionsInserted: actions.length };
|
|
700
|
+
}
|
|
701
|
+
// ===========================================================================
|
|
702
|
+
// Tier 3 — analytics (declared)
|
|
703
|
+
// ===========================================================================
|
|
704
|
+
/**
|
|
705
|
+
* Pass-through for ad-hoc analytical SQL. Appends `FORMAT JSON` if the
|
|
706
|
+
* caller hasn't specified an output format; the parsed `data` array is
|
|
707
|
+
* returned. Use for time-series rollups, aggregations, or callers that
|
|
708
|
+
* want to reach into ClickHouse without going through the adapter's
|
|
709
|
+
* CRUD surface.
|
|
710
|
+
*
|
|
711
|
+
* Note: parameter substitution is left to the caller (ClickHouse's
|
|
712
|
+
* `query_parameters` HTTP shape varies by version). Pre-format the SQL
|
|
713
|
+
* with `quote()` or use `parametrizeQuery` upstream.
|
|
714
|
+
*/
|
|
715
|
+
async analyticsQuery(query, _params) {
|
|
716
|
+
const trimmed = query.trim().replace(/;$/, '');
|
|
717
|
+
const sql = /\bFORMAT\s+\w+\s*$/i.test(trimmed) ? trimmed : `${trimmed} FORMAT JSON`;
|
|
718
|
+
const text = await this.fetcher(sql);
|
|
719
|
+
return parseJsonResponse(text);
|
|
720
|
+
}
|
|
721
|
+
// ===========================================================================
|
|
722
|
+
// Tier 4 — vector search via native distance functions
|
|
723
|
+
// ===========================================================================
|
|
724
|
+
/**
|
|
725
|
+
* Upsert an embedding for a Thing into the `embeddings` table. Backed by
|
|
726
|
+
* `ReplacingMergeTree(version)` so callers may overwrite an embedding
|
|
727
|
+
* by re-inserting; the higher version wins at merge time.
|
|
728
|
+
*
|
|
729
|
+
* @example
|
|
730
|
+
* ```ts
|
|
731
|
+
* await provider.upsertEmbedding('Document', 'doc-1', new Array(1536).fill(0))
|
|
732
|
+
* ```
|
|
733
|
+
*/
|
|
734
|
+
async upsertEmbedding(type, id, embedding) {
|
|
735
|
+
validateTypeName(type);
|
|
736
|
+
validateEntityId(id);
|
|
737
|
+
if (!Array.isArray(embedding) || embedding.length === 0) {
|
|
738
|
+
throw new Error('upsertEmbedding: embedding must be a non-empty array of numbers');
|
|
739
|
+
}
|
|
740
|
+
if (embedding.length > this.vectorDimensions) {
|
|
741
|
+
throw new Error(`upsertEmbedding: embedding length ${embedding.length} exceeds adapter vectorDimensions ${this.vectorDimensions}`);
|
|
742
|
+
}
|
|
743
|
+
for (const v of embedding) {
|
|
744
|
+
if (typeof v !== 'number' || !Number.isFinite(v)) {
|
|
745
|
+
throw new Error('upsertEmbedding: embedding values must be finite numbers');
|
|
746
|
+
}
|
|
747
|
+
}
|
|
748
|
+
const now = new Date();
|
|
749
|
+
const row = {
|
|
750
|
+
ns: this.namespace,
|
|
751
|
+
thing_id: id,
|
|
752
|
+
type,
|
|
753
|
+
embedding: Array.from(embedding),
|
|
754
|
+
version: now.getTime(),
|
|
755
|
+
};
|
|
756
|
+
await this.fetcher(`INSERT INTO ${this.database}.embeddings (ns, thing_id, type, embedding, version) FORMAT JSONEachRow`, JSON.stringify(row) + '\n');
|
|
757
|
+
}
|
|
758
|
+
/**
|
|
759
|
+
* Tier 4 — vector search via ClickHouse native distance functions.
|
|
760
|
+
*
|
|
761
|
+
* Function selection by metric:
|
|
762
|
+
* - `'cosine'` (default): `cosineDistance(embedding, query)` — score is
|
|
763
|
+
* `1 - distance`.
|
|
764
|
+
* - `'l2'`: `L2Distance(embedding, query)` — score is `-distance`.
|
|
765
|
+
* - `'dot'`: `dotProduct(embedding, query)` — score is the inner product
|
|
766
|
+
* directly.
|
|
767
|
+
*
|
|
768
|
+
* Frame-aware role filtering is deferred (see PG adapter doc).
|
|
769
|
+
*/
|
|
770
|
+
async vectorSearch(type, queryEmbedding, options) {
|
|
771
|
+
validateTypeName(type);
|
|
772
|
+
if (!Array.isArray(queryEmbedding) || queryEmbedding.length === 0) {
|
|
773
|
+
throw new Error('vectorSearch: queryEmbedding must be a non-empty array of numbers');
|
|
774
|
+
}
|
|
775
|
+
if (queryEmbedding.length > this.vectorDimensions) {
|
|
776
|
+
throw new Error(`vectorSearch: query length ${queryEmbedding.length} exceeds adapter vectorDimensions ${this.vectorDimensions}`);
|
|
777
|
+
}
|
|
778
|
+
for (const v of queryEmbedding) {
|
|
779
|
+
if (typeof v !== 'number' || !Number.isFinite(v)) {
|
|
780
|
+
throw new Error('vectorSearch: queryEmbedding values must be finite numbers');
|
|
781
|
+
}
|
|
782
|
+
}
|
|
783
|
+
const metric = options?.metric ?? 'cosine';
|
|
784
|
+
const limit = Math.max(1, options?.limit ?? 10);
|
|
785
|
+
let scoreExpr;
|
|
786
|
+
let orderExpr;
|
|
787
|
+
switch (metric) {
|
|
788
|
+
case 'cosine':
|
|
789
|
+
scoreExpr = `1 - cosineDistance(e.embedding, [${queryEmbedding.join(',')}])`;
|
|
790
|
+
orderExpr = `cosineDistance(e.embedding, [${queryEmbedding.join(',')}]) ASC`;
|
|
791
|
+
break;
|
|
792
|
+
case 'l2':
|
|
793
|
+
scoreExpr = `-L2Distance(e.embedding, [${queryEmbedding.join(',')}])`;
|
|
794
|
+
orderExpr = `L2Distance(e.embedding, [${queryEmbedding.join(',')}]) ASC`;
|
|
795
|
+
break;
|
|
796
|
+
case 'dot':
|
|
797
|
+
scoreExpr = `dotProduct(e.embedding, [${queryEmbedding.join(',')}])`;
|
|
798
|
+
orderExpr = `dotProduct(e.embedding, [${queryEmbedding.join(',')}]) DESC`;
|
|
799
|
+
break;
|
|
800
|
+
case 'hamming':
|
|
801
|
+
throw new Error('vectorSearch: ClickHouse adapter does not support hamming metric');
|
|
802
|
+
default:
|
|
803
|
+
throw new Error(`vectorSearch: unsupported metric "${String(metric)}"`);
|
|
804
|
+
}
|
|
805
|
+
const sql = `SELECT t.id AS id, t.type AS type, t.data AS data, ${scoreExpr} AS score
|
|
806
|
+
FROM ${this.database}.embeddings FINAL e
|
|
807
|
+
INNER JOIN ${this.database}.things FINAL t
|
|
808
|
+
ON t.ns = e.ns AND t.id = e.thing_id
|
|
809
|
+
WHERE e.ns = ${quote(this.namespace)} AND t.type = ${quote(type)}
|
|
810
|
+
ORDER BY ${orderExpr}
|
|
811
|
+
LIMIT ${Number(limit)}
|
|
812
|
+
FORMAT JSON`;
|
|
813
|
+
const text = await this.fetcher(sql);
|
|
814
|
+
const rows = parseJsonResponse(text);
|
|
815
|
+
let hits = rows.map((row) => {
|
|
816
|
+
const data = asJsonb(row['data']);
|
|
817
|
+
const rawScore = row['score'];
|
|
818
|
+
const score = typeof rawScore === 'number' ? rawScore : Number(rawScore ?? 0);
|
|
819
|
+
return {
|
|
820
|
+
entity: { ...data, $id: String(row['id']), $type: String(row['type']) },
|
|
821
|
+
score,
|
|
822
|
+
};
|
|
823
|
+
});
|
|
824
|
+
if (options?.minScore !== undefined) {
|
|
825
|
+
const min = options.minScore;
|
|
826
|
+
hits = hits.filter((h) => h.score >= min);
|
|
827
|
+
}
|
|
828
|
+
return hits;
|
|
829
|
+
}
|
|
830
|
+
/**
|
|
831
|
+
* Driver / connection metadata for diagnostics.
|
|
832
|
+
*/
|
|
833
|
+
describe() {
|
|
834
|
+
return {
|
|
835
|
+
adapter: 'clickhouse',
|
|
836
|
+
driver: this.driver,
|
|
837
|
+
namespace: this.namespace,
|
|
838
|
+
database: this.database,
|
|
839
|
+
};
|
|
840
|
+
}
|
|
841
|
+
}
|
|
842
|
+
// =============================================================================
|
|
843
|
+
// Schema bootstrap
|
|
844
|
+
// =============================================================================
|
|
845
|
+
/**
|
|
846
|
+
* Run the canonical DDL against a ClickHouse fetcher. Idempotent — uses
|
|
847
|
+
* `IF NOT EXISTS` throughout. Designed for one-shot cluster bootstrap and
|
|
848
|
+
* for test harnesses; production deployments typically run schema
|
|
849
|
+
* migrations via a CI tool.
|
|
850
|
+
*/
|
|
851
|
+
export async function bootstrapClickHouseSchema(fetcher, options = {}) {
|
|
852
|
+
const database = options.database ?? DEFAULT_DATABASE;
|
|
853
|
+
// vectorDimensions is informational — Array(Float32) is dynamically
|
|
854
|
+
// sized in CH; we capture it for future-proofing (e.g. switching to
|
|
855
|
+
// fixed Tuple types) but no DDL pin needed today.
|
|
856
|
+
void options.vectorDimensions;
|
|
857
|
+
await fetcher(`CREATE DATABASE IF NOT EXISTS ${database}`);
|
|
858
|
+
await fetcher(`CREATE TABLE IF NOT EXISTS ${database}.things (
|
|
859
|
+
ns String,
|
|
860
|
+
id String,
|
|
861
|
+
type String,
|
|
862
|
+
data String,
|
|
863
|
+
created_at DateTime64(3) DEFAULT now64(3),
|
|
864
|
+
updated_at DateTime64(3) DEFAULT now64(3),
|
|
865
|
+
version UInt64 DEFAULT toUnixTimestamp64Milli(now64(3))
|
|
866
|
+
) ENGINE = ReplacingMergeTree(version)
|
|
867
|
+
ORDER BY (ns, type, id)`);
|
|
868
|
+
await fetcher(`CREATE TABLE IF NOT EXISTS ${database}.actions (
|
|
869
|
+
ns String,
|
|
870
|
+
id String,
|
|
871
|
+
verb String,
|
|
872
|
+
subject String,
|
|
873
|
+
object String,
|
|
874
|
+
roles String,
|
|
875
|
+
data String,
|
|
876
|
+
status LowCardinality(String) DEFAULT 'pending',
|
|
877
|
+
created_at DateTime64(3) DEFAULT now64(3),
|
|
878
|
+
completed_at Nullable(DateTime64(3))
|
|
879
|
+
) ENGINE = MergeTree
|
|
880
|
+
PARTITION BY toYYYYMM(created_at)
|
|
881
|
+
ORDER BY (ns, verb, subject, created_at)`);
|
|
882
|
+
await fetcher(`CREATE TABLE IF NOT EXISTS ${database}.verbs (
|
|
883
|
+
name String,
|
|
884
|
+
data String,
|
|
885
|
+
created_at DateTime64(3) DEFAULT now64(3),
|
|
886
|
+
version UInt64 DEFAULT toUnixTimestamp64Milli(now64(3))
|
|
887
|
+
) ENGINE = ReplacingMergeTree(version)
|
|
888
|
+
ORDER BY name`);
|
|
889
|
+
// Tier 4 — embeddings table. Array(Float32) accepts variable-length
|
|
890
|
+
// vectors, which keeps the schema portable across embedding models. The
|
|
891
|
+
// adapter validates dimension against `vectorDimensions` at the API
|
|
892
|
+
// surface; CH itself does not pin the dimension.
|
|
893
|
+
await fetcher(`CREATE TABLE IF NOT EXISTS ${database}.embeddings (
|
|
894
|
+
ns String,
|
|
895
|
+
thing_id String,
|
|
896
|
+
type String,
|
|
897
|
+
embedding Array(Float32),
|
|
898
|
+
version UInt64 DEFAULT toUnixTimestamp64Milli(now64(3))
|
|
899
|
+
) ENGINE = ReplacingMergeTree(version)
|
|
900
|
+
ORDER BY (ns, type, thing_id)`);
|
|
901
|
+
// Hint to keep DBProvider import warm.
|
|
902
|
+
void true;
|
|
903
|
+
}
|
|
904
|
+
// =============================================================================
|
|
905
|
+
// Factory
|
|
906
|
+
// =============================================================================
|
|
907
|
+
/**
|
|
908
|
+
* Convenience factory for {@link ClickHouseProvider}.
|
|
909
|
+
*
|
|
910
|
+
* @example
|
|
911
|
+
* ```ts
|
|
912
|
+
* import { createClickHouseProvider, createClickHouseHttpFetcher } from 'ai-database'
|
|
913
|
+
*
|
|
914
|
+
* const fetcher = createClickHouseHttpFetcher({
|
|
915
|
+
* url: env.CLICKHOUSE_URL,
|
|
916
|
+
* username: env.CLICKHOUSE_USER,
|
|
917
|
+
* password: env.CLICKHOUSE_PASSWORD,
|
|
918
|
+
* database: 'aidb',
|
|
919
|
+
* })
|
|
920
|
+
* const provider = createClickHouseProvider({
|
|
921
|
+
* fetcher,
|
|
922
|
+
* namespace: 'tenant-9',
|
|
923
|
+
* })
|
|
924
|
+
* ```
|
|
925
|
+
*/
|
|
926
|
+
export function createClickHouseProvider(options) {
|
|
927
|
+
return new ClickHouseProvider(options);
|
|
928
|
+
}
|
|
929
|
+
//# sourceMappingURL=ch-adapter.js.map
|