@notuslabs/yail 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +226 -0
- package/bin/yail.js +7 -0
- package/dist/addresses/registry.d.ts +57 -0
- package/dist/addresses/registry.d.ts.map +1 -0
- package/dist/addresses/registry.js +131 -0
- package/dist/addresses/registry.js.map +1 -0
- package/dist/cli/index.d.ts +2 -0
- package/dist/cli/index.d.ts.map +1 -0
- package/dist/cli/index.js +204 -0
- package/dist/cli/index.js.map +1 -0
- package/dist/config/address.d.ts +32 -0
- package/dist/config/address.d.ts.map +1 -0
- package/dist/config/address.js +31 -0
- package/dist/config/address.js.map +1 -0
- package/dist/config/chain-ref.d.ts +5 -0
- package/dist/config/chain-ref.d.ts.map +1 -0
- package/dist/config/chain-ref.js +13 -0
- package/dist/config/chain-ref.js.map +1 -0
- package/dist/config/create-config.d.ts +9 -0
- package/dist/config/create-config.d.ts.map +1 -0
- package/dist/config/create-config.js +86 -0
- package/dist/config/create-config.js.map +1 -0
- package/dist/config/index.d.ts +5 -0
- package/dist/config/index.d.ts.map +1 -0
- package/dist/config/index.js +3 -0
- package/dist/config/index.js.map +1 -0
- package/dist/config/types.d.ts +113 -0
- package/dist/config/types.d.ts.map +1 -0
- package/dist/config/types.js +2 -0
- package/dist/config/types.js.map +1 -0
- package/dist/db/batch.d.ts +33 -0
- package/dist/db/batch.d.ts.map +1 -0
- package/dist/db/batch.js +71 -0
- package/dist/db/batch.js.map +1 -0
- package/dist/db/client.d.ts +50 -0
- package/dist/db/client.d.ts.map +1 -0
- package/dist/db/client.js +104 -0
- package/dist/db/client.js.map +1 -0
- package/dist/db/index.d.ts +11 -0
- package/dist/db/index.d.ts.map +1 -0
- package/dist/db/index.js +7 -0
- package/dist/db/index.js.map +1 -0
- package/dist/db/internal.d.ts +2332 -0
- package/dist/db/internal.d.ts.map +1 -0
- package/dist/db/internal.js +75 -0
- package/dist/db/internal.js.map +1 -0
- package/dist/db/migrate.d.ts +22 -0
- package/dist/db/migrate.d.ts.map +1 -0
- package/dist/db/migrate.js +72 -0
- package/dist/db/migrate.js.map +1 -0
- package/dist/db/serialize.d.ts +14 -0
- package/dist/db/serialize.d.ts.map +1 -0
- package/dist/db/serialize.js +133 -0
- package/dist/db/serialize.js.map +1 -0
- package/dist/db/sql.d.ts +44 -0
- package/dist/db/sql.d.ts.map +1 -0
- package/dist/db/sql.js +122 -0
- package/dist/db/sql.js.map +1 -0
- package/dist/http/client.d.ts +40 -0
- package/dist/http/client.d.ts.map +1 -0
- package/dist/http/client.js +128 -0
- package/dist/http/client.js.map +1 -0
- package/dist/index.d.ts +21 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +15 -0
- package/dist/index.js.map +1 -0
- package/dist/indexer/context.d.ts +84 -0
- package/dist/indexer/context.d.ts.map +1 -0
- package/dist/indexer/context.js +45 -0
- package/dist/indexer/context.js.map +1 -0
- package/dist/indexer/events.d.ts +87 -0
- package/dist/indexer/events.d.ts.map +1 -0
- package/dist/indexer/events.js +88 -0
- package/dist/indexer/events.js.map +1 -0
- package/dist/indexer/indexer.d.ts +114 -0
- package/dist/indexer/indexer.d.ts.map +1 -0
- package/dist/indexer/indexer.js +237 -0
- package/dist/indexer/indexer.js.map +1 -0
- package/dist/indexer/jobs.d.ts +61 -0
- package/dist/indexer/jobs.d.ts.map +1 -0
- package/dist/indexer/jobs.js +170 -0
- package/dist/indexer/jobs.js.map +1 -0
- package/dist/indexer/plan.d.ts +73 -0
- package/dist/indexer/plan.d.ts.map +1 -0
- package/dist/indexer/plan.js +238 -0
- package/dist/indexer/plan.js.map +1 -0
- package/dist/indexer/runtime.d.ts +100 -0
- package/dist/indexer/runtime.d.ts.map +1 -0
- package/dist/indexer/runtime.js +572 -0
- package/dist/indexer/runtime.js.map +1 -0
- package/dist/observability/logger.d.ts +23 -0
- package/dist/observability/logger.d.ts.map +1 -0
- package/dist/observability/logger.js +72 -0
- package/dist/observability/logger.js.map +1 -0
- package/dist/observability/metrics.d.ts +61 -0
- package/dist/observability/metrics.d.ts.map +1 -0
- package/dist/observability/metrics.js +131 -0
- package/dist/observability/metrics.js.map +1 -0
- package/dist/observability/otel.d.ts +28 -0
- package/dist/observability/otel.d.ts.map +1 -0
- package/dist/observability/otel.js +49 -0
- package/dist/observability/otel.js.map +1 -0
- package/dist/rpc/cached-client.d.ts +33 -0
- package/dist/rpc/cached-client.d.ts.map +1 -0
- package/dist/rpc/cached-client.js +50 -0
- package/dist/rpc/cached-client.js.map +1 -0
- package/dist/schema/column.d.ts +61 -0
- package/dist/schema/column.d.ts.map +1 -0
- package/dist/schema/column.js +76 -0
- package/dist/schema/column.js.map +1 -0
- package/dist/schema/ddl.d.ts +9 -0
- package/dist/schema/ddl.d.ts.map +1 -0
- package/dist/schema/ddl.js +58 -0
- package/dist/schema/ddl.js.map +1 -0
- package/dist/schema/index.d.ts +8 -0
- package/dist/schema/index.d.ts.map +1 -0
- package/dist/schema/index.js +5 -0
- package/dist/schema/index.js.map +1 -0
- package/dist/schema/table.d.ts +69 -0
- package/dist/schema/table.d.ts.map +1 -0
- package/dist/schema/table.js +35 -0
- package/dist/schema/table.js.map +1 -0
- package/dist/schema/view.d.ts +19 -0
- package/dist/schema/view.d.ts.map +1 -0
- package/dist/schema/view.js +16 -0
- package/dist/schema/view.js.map +1 -0
- package/dist/server/api.d.ts +11 -0
- package/dist/server/api.d.ts.map +1 -0
- package/dist/server/api.js +71 -0
- package/dist/server/api.js.map +1 -0
- package/dist/sources/cached.d.ts +27 -0
- package/dist/sources/cached.d.ts.map +1 -0
- package/dist/sources/cached.js +111 -0
- package/dist/sources/cached.js.map +1 -0
- package/dist/sources/esplora.d.ts +22 -0
- package/dist/sources/esplora.d.ts.map +1 -0
- package/dist/sources/esplora.js +138 -0
- package/dist/sources/esplora.js.map +1 -0
- package/dist/sources/fixture.d.ts +51 -0
- package/dist/sources/fixture.d.ts.map +1 -0
- package/dist/sources/fixture.js +102 -0
- package/dist/sources/fixture.js.map +1 -0
- package/dist/sources/hypersync.d.ts +19 -0
- package/dist/sources/hypersync.d.ts.map +1 -0
- package/dist/sources/hypersync.js +169 -0
- package/dist/sources/hypersync.js.map +1 -0
- package/dist/sources/index.d.ts +13 -0
- package/dist/sources/index.d.ts.map +1 -0
- package/dist/sources/index.js +7 -0
- package/dist/sources/index.js.map +1 -0
- package/dist/sources/rpc.d.ts +21 -0
- package/dist/sources/rpc.d.ts.map +1 -0
- package/dist/sources/rpc.js +187 -0
- package/dist/sources/rpc.js.map +1 -0
- package/dist/sources/types.d.ts +137 -0
- package/dist/sources/types.d.ts.map +1 -0
- package/dist/sources/types.js +17 -0
- package/dist/sources/types.js.map +1 -0
- package/dist/testing/index.d.ts +20 -0
- package/dist/testing/index.d.ts.map +1 -0
- package/dist/testing/index.js +44 -0
- package/dist/testing/index.js.map +1 -0
- package/dist/util.d.ts +9 -0
- package/dist/util.d.ts.map +1 -0
- package/dist/util.js +25 -0
- package/dist/util.js.map +1 -0
- package/package.json +100 -0
package/dist/db/batch.js
ADDED
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
import { serializeRow } from "./serialize.js";
|
|
2
|
+
/**
|
|
3
|
+
* Buffers rows per table and flushes them in one INSERT per table.
|
|
4
|
+
* ClickHouse wants few, large inserts; handlers want to write row-by-row.
|
|
5
|
+
* The indexer flushes after every source batch, before writing the checkpoint.
|
|
6
|
+
*/
|
|
7
|
+
export class BatchWriter {
|
|
8
|
+
db;
|
|
9
|
+
options;
|
|
10
|
+
pending = new Map();
|
|
11
|
+
rowCount = 0;
|
|
12
|
+
constructor(db, options = {}) {
|
|
13
|
+
this.db = db;
|
|
14
|
+
this.options = options;
|
|
15
|
+
}
|
|
16
|
+
get size() {
|
|
17
|
+
return this.rowCount;
|
|
18
|
+
}
|
|
19
|
+
add(table, row, meta) {
|
|
20
|
+
let p = this.pending.get(table.name);
|
|
21
|
+
if (!p) {
|
|
22
|
+
p = { table, rows: [], byKey: new Map() };
|
|
23
|
+
this.pending.set(table.name, p);
|
|
24
|
+
}
|
|
25
|
+
p.rows.push(serializeRow(table, row, meta));
|
|
26
|
+
p.byKey.set(keyOf(table, row), row);
|
|
27
|
+
this.rowCount++;
|
|
28
|
+
}
|
|
29
|
+
/** Latest buffered logical row matching the full ORDER BY key, if any. */
|
|
30
|
+
peek(table, key) {
|
|
31
|
+
const p = this.pending.get(table.name);
|
|
32
|
+
if (!p)
|
|
33
|
+
return null;
|
|
34
|
+
const complete = table.options.orderBy.every((k) => key[k] !== undefined);
|
|
35
|
+
if (!complete)
|
|
36
|
+
return null;
|
|
37
|
+
return p.byKey.get(keyOf(table, key)) ?? null;
|
|
38
|
+
}
|
|
39
|
+
async flush() {
|
|
40
|
+
const entries = [...this.pending.values()];
|
|
41
|
+
this.pending = new Map();
|
|
42
|
+
const rows = this.rowCount;
|
|
43
|
+
this.rowCount = 0;
|
|
44
|
+
for (const p of entries) {
|
|
45
|
+
const start = performance.now();
|
|
46
|
+
await this.db.insertRaw(p.table.name, p.rows);
|
|
47
|
+
this.options.onFlush?.({ table: p.table.name, rows: p.rows.length, ms: performance.now() - start });
|
|
48
|
+
}
|
|
49
|
+
return { tables: entries.length, rows };
|
|
50
|
+
}
|
|
51
|
+
shouldFlush() {
|
|
52
|
+
return this.options.maxRows !== undefined && this.rowCount >= this.options.maxRows;
|
|
53
|
+
}
|
|
54
|
+
discard() {
|
|
55
|
+
this.pending = new Map();
|
|
56
|
+
this.rowCount = 0;
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
function keyOf(table, row) {
|
|
60
|
+
return table.options.orderBy
|
|
61
|
+
.map((k) => {
|
|
62
|
+
const v = row[k];
|
|
63
|
+
if (typeof v === "string" && v.startsWith("0x"))
|
|
64
|
+
return v.toLowerCase();
|
|
65
|
+
if (v instanceof Date)
|
|
66
|
+
return String(v.getTime());
|
|
67
|
+
return String(v);
|
|
68
|
+
})
|
|
69
|
+
.join("\u0001");
|
|
70
|
+
}
|
|
71
|
+
//# sourceMappingURL=batch.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"batch.js","sourceRoot":"","sources":["../../src/db/batch.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,YAAY,EAAgB,MAAM,gBAAgB,CAAC;AAS5D;;;;GAIG;AACH,MAAM,OAAO,WAAW;IAKH,EAAE;IACF,OAAO;IALlB,OAAO,GAAG,IAAI,GAAG,EAAmB,CAAC;IACrC,QAAQ,GAAG,CAAC,CAAC;IAErB,YACmB,EAAM,EACN,OAAO,GAAiG,EAAE;kBAD1G,EAAE;uBACF,OAAO;IACvB,CAAC;IAEJ,IAAI,IAAI;QACN,OAAO,IAAI,CAAC,QAAQ,CAAC;IACvB,CAAC;IAED,GAAG,CAAuB,KAAQ,EAAE,GAAiB,EAAE,IAAc;QACnE,IAAI,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACrC,IAAI,CAAC,CAAC,EAAE,CAAC;YACP,CAAC,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE,EAAE,KAAK,EAAE,IAAI,GAAG,EAAE,EAAE,CAAC;YAC1C,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAClC,CAAC;QACD,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,GAA8B,EAAE,IAAI,CAAC,CAAC,CAAC;QACvE,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,EAAE,GAA8B,CAAC,EAAE,GAA8B,CAAC,CAAC;QAC1F,IAAI,CAAC,QAAQ,EAAE,CAAC;IAClB,CAAC;IAED,0EAA0E;IAC1E,IAAI,CAAuB,KAAQ,EAAE,GAAyB;QAC5D,MAAM,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACvC,IAAI,CAAC,CAAC;YAAE,OAAO,IAAI,CAAC;QACpB,MAAM,QAAQ,GAAG,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAE,GAAW,CAAC,CAAC,CAAC,KAAK,SAAS,CAAC,CAAC;QACnF,IAAI,CAAC,QAAQ;YAAE,OAAO,IAAI,CAAC;QAC3B,OAAQ,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,EAAE,GAA8B,CAAC,CAAiB,IAAI,IAAI,CAAC;IAC5F,CAAC;IAED,KAAK,CAAC,KAAK;QACT,MAAM,OAAO,GAAG,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;QAC3C,IAAI,CAAC,OAAO,GAAG,IAAI,GAAG,EAAE,CAAC;QACzB,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC;QAC3B,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;QAClB,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACxB,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;YAChC,MAAM,IAAI,CAAC,EAAE,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;YAC9C,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,EAAE,WAAW,CAAC,GAAG,EAAE,GAAG,KAAK,EAAE,CAAC,CAAC;QACtG,CAAC;QACD,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC;IAC1C,CAAC;IAED,WAAW;QACT,OAAO,IAAI,CAAC,OAAO,CAAC,OAAO,KAAK,SAAS,IAAI,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC;IACrF,CAAC;IAED,OAAO;QACL,IAAI,CAAC,OAAO,GAAG,IAAI,GAAG,EAAE,CAAC;QACzB,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;IACpB,CAAC;CACF;AAED,SAAS,KAAK,CAAC,KAAiB,EAAE,GAA4B;IAC5D,OAAO,KAAK,CAAC,OAAO,CAAC,OAAO;SACzB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACT,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;QACjB,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC;YAAE,OAAO,CAAC,CAAC,WAAW,EAAE,CAAC;QACxE,IAAI,CAAC,YAAY,IAAI;YAAE,OAAO,MAAM,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;QAClD,OAAO,MAAM,CAAC,CAAC,CAAC,CAAC;IACnB,CAAC,CAAC;SACD,IAAI,CAAC,QAAQ,CAAC,CAAC;AACpB,CAAC"}
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import { type ClickHouseClient } from "@clickhouse/client";
|
|
2
|
+
import type { InferRow, InsertRow, Table } from "../schema/table.js";
|
|
3
|
+
import { type SqlFragment } from "./sql.js";
|
|
4
|
+
import { type RowMeta } from "./serialize.js";
|
|
5
|
+
export interface DatabaseConfig {
|
|
6
|
+
/** ClickHouse HTTP URL, e.g. http://localhost:8123. Credentials may be embedded (http://user:pass@host:8123). */
|
|
7
|
+
url: string;
|
|
8
|
+
/** Database name. Created by `migrate` if missing. Default `default`. */
|
|
9
|
+
database?: string;
|
|
10
|
+
username?: string;
|
|
11
|
+
password?: string;
|
|
12
|
+
/** Extra ClickHouse settings applied to every request. */
|
|
13
|
+
settings?: Record<string, string | number | boolean>;
|
|
14
|
+
/** Request timeout in ms. Default 60s. */
|
|
15
|
+
requestTimeout?: number;
|
|
16
|
+
}
|
|
17
|
+
export interface QueryOptions {
|
|
18
|
+
settings?: Record<string, string | number | boolean>;
|
|
19
|
+
signal?: AbortSignal;
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* Thin, typed ClickHouse client. This is the same object handlers receive as
|
|
23
|
+
* `context.db` and the same one an API can use to read what the indexer wrote.
|
|
24
|
+
*/
|
|
25
|
+
export interface Db {
|
|
26
|
+
readonly client: ClickHouseClient;
|
|
27
|
+
readonly database: string;
|
|
28
|
+
/** Run a SELECT and return JSONEachRow rows. Prefer `sql` fragments for parameters. */
|
|
29
|
+
query<T = Record<string, unknown>>(query: SqlFragment | string, options?: QueryOptions): Promise<T[]>;
|
|
30
|
+
/** Run a statement that returns no rows (DDL, DELETE, INSERT ... SELECT). */
|
|
31
|
+
command(query: SqlFragment | string, options?: QueryOptions): Promise<void>;
|
|
32
|
+
/** Typed rows of a schema table. `where` is appended after `FROM table [FINAL]`. */
|
|
33
|
+
rows<T extends Table<any>>(table: T, where?: SqlFragment, options?: {
|
|
34
|
+
final?: boolean;
|
|
35
|
+
} & QueryOptions): Promise<Array<InferRow<T>>>;
|
|
36
|
+
/** Find one row by (a subset of) its ORDER BY key. Reads with FINAL so re-inserted rows collapse. */
|
|
37
|
+
find<T extends Table<any>>(table: T, key: Partial<InferRow<T>>): Promise<InferRow<T> | null>;
|
|
38
|
+
/** Immediate (unbuffered) insert. Handlers should use `context.db.insert` which is batched. */
|
|
39
|
+
insert<T extends Table<any>>(table: T): {
|
|
40
|
+
values(rows: InsertRow<T> | InsertRow<T>[], meta?: RowMeta): Promise<void>;
|
|
41
|
+
};
|
|
42
|
+
/** Low-level insert of pre-serialized JSONEachRow rows. */
|
|
43
|
+
insertRaw(tableName: string, rows: Record<string, unknown>[], options?: QueryOptions): Promise<void>;
|
|
44
|
+
/** Create the configured database if missing (uses a connection bound to `default`). */
|
|
45
|
+
createDatabase(): Promise<void>;
|
|
46
|
+
ping(): Promise<boolean>;
|
|
47
|
+
close(): Promise<void>;
|
|
48
|
+
}
|
|
49
|
+
export declare function createDb(config: DatabaseConfig): Db;
|
|
50
|
+
//# sourceMappingURL=client.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../../src/db/client.ts"],"names":[],"mappings":"AAAA,OAAO,EAAgB,KAAK,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACzE,OAAO,KAAK,EAAE,QAAQ,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AACrE,OAAO,EAAkB,KAAK,WAAW,EAAE,MAAM,UAAU,CAAC;AAC5D,OAAO,EAAgC,KAAK,OAAO,EAAE,MAAM,gBAAgB,CAAC;AAG5E,MAAM,WAAW,cAAc;IAC7B,iHAAiH;IACjH,GAAG,EAAE,MAAM,CAAC;IACZ,yEAAyE;IACzE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,0DAA0D;IAC1D,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC,CAAC;IACrD,0CAA0C;IAC1C,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC,CAAC;IACrD,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB;AAED;;;GAGG;AACH,MAAM,WAAW,EAAE;IACjB,QAAQ,CAAC,MAAM,EAAE,gBAAgB,CAAC;IAClC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,uFAAuF;IACvF,KAAK,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,KAAK,EAAE,WAAW,GAAG,MAAM,EAAE,OAAO,CAAC,EAAE,YAAY,GAAG,OAAO,CAAC,CAAC,EAAE,CAAC,CAAC;IACtG,6EAA6E;IAC7E,OAAO,CAAC,KAAK,EAAE,WAAW,GAAG,MAAM,EAAE,OAAO,CAAC,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5E,oFAAoF;IACpF,IAAI,CAAC,CAAC,SAAS,KAAK,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,CAAC,EAAE,WAAW,EAAE,OAAO,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,OAAO,CAAA;KAAE,GAAG,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACrI,qGAAqG;IACrG,IAAI,CAAC,CAAC,SAAS,KAAK,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;IAC7F,+FAA+F;IAC/F,MAAM,CAAC,CAAC,SAAS,KAAK,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,CAAC,GAAG;QAAE,MAAM,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,IAAI,CAAC,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;KAAE,CAAC;IACvH,2DAA2D;IAC3D,SAAS,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,EAAE,OAAO,CAAC,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACrG,wFAAwF;IACxF,cAAc,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAChC,IAAI,IAAI,OAAO,CAAC,OAAO,CAAC,CAAC;IACzB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACxB;AAQD,wBAAgB,QAAQ,CAAC,MAAM,EAAE,cAAc,GAAG,EAAE,CA6FnD"}
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
import { createClient } from "@clickhouse/client";
|
|
2
|
+
import { renderSql, sql } from "./sql.js";
|
|
3
|
+
import { deserializeRow, serializeRow } from "./serialize.js";
|
|
4
|
+
import { decodeOr, toArray } from "../util.js";
|
|
5
|
+
const DEFAULT_SETTINGS = {
|
|
6
|
+
date_time_output_format: "iso",
|
|
7
|
+
output_format_json_quote_64bit_integers: 1,
|
|
8
|
+
input_format_import_nested_json: 0,
|
|
9
|
+
};
|
|
10
|
+
export function createDb(config) {
|
|
11
|
+
const parsed = new URL(config.url);
|
|
12
|
+
const username = config.username ?? decodeOr(parsed.username, "default");
|
|
13
|
+
const password = config.password ?? decodeOr(parsed.password, "");
|
|
14
|
+
const database = config.database ?? (parsed.pathname.replace(/^\//, "") || "default");
|
|
15
|
+
parsed.username = "";
|
|
16
|
+
parsed.password = "";
|
|
17
|
+
parsed.pathname = "/";
|
|
18
|
+
const client = createClient({
|
|
19
|
+
url: parsed.toString(),
|
|
20
|
+
username,
|
|
21
|
+
password,
|
|
22
|
+
database,
|
|
23
|
+
request_timeout: config.requestTimeout ?? 60_000,
|
|
24
|
+
clickhouse_settings: { ...DEFAULT_SETTINGS, ...(config.settings ?? {}) },
|
|
25
|
+
});
|
|
26
|
+
const db = {
|
|
27
|
+
client,
|
|
28
|
+
database,
|
|
29
|
+
async query(query, options) {
|
|
30
|
+
const { text, params } = renderSql(query);
|
|
31
|
+
const rs = await client.query({
|
|
32
|
+
query: text,
|
|
33
|
+
query_params: params,
|
|
34
|
+
format: "JSONEachRow",
|
|
35
|
+
clickhouse_settings: options?.settings,
|
|
36
|
+
abort_signal: options?.signal,
|
|
37
|
+
});
|
|
38
|
+
return (await rs.json());
|
|
39
|
+
},
|
|
40
|
+
async command(query, options) {
|
|
41
|
+
const { text, params } = renderSql(query);
|
|
42
|
+
await client.command({
|
|
43
|
+
query: text,
|
|
44
|
+
query_params: params,
|
|
45
|
+
clickhouse_settings: options?.settings,
|
|
46
|
+
abort_signal: options?.signal,
|
|
47
|
+
});
|
|
48
|
+
},
|
|
49
|
+
async rows(table, where, options) {
|
|
50
|
+
let finalKeyword = " FINAL";
|
|
51
|
+
if (options?.final === false)
|
|
52
|
+
finalKeyword = "";
|
|
53
|
+
const q = sql `SELECT * FROM ${table}${sql.raw(finalKeyword)} ${where ?? sql.empty()}`;
|
|
54
|
+
const raw = await db.query(q, options);
|
|
55
|
+
return raw.map((r) => deserializeRow(table, r));
|
|
56
|
+
},
|
|
57
|
+
async find(table, key) {
|
|
58
|
+
const conds = Object.entries(key).map(([k, v]) => sql `${sql.identifier(table.columnName(k))} = ${v}`);
|
|
59
|
+
if (conds.length === 0)
|
|
60
|
+
throw new Error("find() requires at least one key column");
|
|
61
|
+
const rows = await db.rows(table, sql `WHERE ${sql.join(conds, " AND ")} LIMIT 1`);
|
|
62
|
+
return rows[0] ?? null;
|
|
63
|
+
},
|
|
64
|
+
insert(table) {
|
|
65
|
+
return {
|
|
66
|
+
async values(rows, meta) {
|
|
67
|
+
const list = toArray(rows);
|
|
68
|
+
if (list.length === 0)
|
|
69
|
+
return;
|
|
70
|
+
await db.insertRaw(table.name, list.map((r) => serializeRow(table, r, meta)));
|
|
71
|
+
},
|
|
72
|
+
};
|
|
73
|
+
},
|
|
74
|
+
async insertRaw(tableName, rows, options) {
|
|
75
|
+
if (rows.length === 0)
|
|
76
|
+
return;
|
|
77
|
+
await client.insert({
|
|
78
|
+
table: tableName,
|
|
79
|
+
values: rows,
|
|
80
|
+
format: "JSONEachRow",
|
|
81
|
+
clickhouse_settings: options?.settings,
|
|
82
|
+
abort_signal: options?.signal,
|
|
83
|
+
});
|
|
84
|
+
},
|
|
85
|
+
async createDatabase() {
|
|
86
|
+
const admin = createClient({ url: parsed.toString(), username, password, database: "default", request_timeout: config.requestTimeout ?? 60_000 });
|
|
87
|
+
try {
|
|
88
|
+
await admin.command({ query: `CREATE DATABASE IF NOT EXISTS \`${database}\`` });
|
|
89
|
+
}
|
|
90
|
+
finally {
|
|
91
|
+
await admin.close();
|
|
92
|
+
}
|
|
93
|
+
},
|
|
94
|
+
async ping() {
|
|
95
|
+
const r = await client.ping();
|
|
96
|
+
return r.success;
|
|
97
|
+
},
|
|
98
|
+
async close() {
|
|
99
|
+
await client.close();
|
|
100
|
+
},
|
|
101
|
+
};
|
|
102
|
+
return db;
|
|
103
|
+
}
|
|
104
|
+
//# sourceMappingURL=client.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"client.js","sourceRoot":"","sources":["../../src/db/client.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAyB,MAAM,oBAAoB,CAAC;AAEzE,OAAO,EAAE,SAAS,EAAE,GAAG,EAAoB,MAAM,UAAU,CAAC;AAC5D,OAAO,EAAE,cAAc,EAAE,YAAY,EAAgB,MAAM,gBAAgB,CAAC;AAC5E,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AA6C/C,MAAM,gBAAgB,GAAG;IACvB,uBAAuB,EAAE,KAAK;IAC9B,uCAAuC,EAAE,CAAC;IAC1C,+BAA+B,EAAE,CAAC;CAC1B,CAAC;AAEX,MAAM,UAAU,QAAQ,CAAC,MAAsB;IAC7C,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IACnC,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,IAAI,QAAQ,CAAC,MAAM,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;IACzE,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,IAAI,QAAQ,CAAC,MAAM,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;IAClE,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,SAAS,CAAC,CAAC;IACtF,MAAM,CAAC,QAAQ,GAAG,EAAE,CAAC;IACrB,MAAM,CAAC,QAAQ,GAAG,EAAE,CAAC;IACrB,MAAM,CAAC,QAAQ,GAAG,GAAG,CAAC;IAEtB,MAAM,MAAM,GAAG,YAAY,CAAC;QAC1B,GAAG,EAAE,MAAM,CAAC,QAAQ,EAAE;QACtB,QAAQ;QACR,QAAQ;QACR,QAAQ;QACR,eAAe,EAAE,MAAM,CAAC,cAAc,IAAI,MAAM;QAChD,mBAAmB,EAAE,EAAE,GAAG,gBAAgB,EAAE,GAAG,CAAC,MAAM,CAAC,QAAQ,IAAI,EAAE,CAAC,EAAS;KAChF,CAAC,CAAC;IAEH,MAAM,EAAE,GAAO;QACb,MAAM;QACN,QAAQ;QACR,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,OAAO;YACxB,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC;YAC1C,MAAM,EAAE,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC;gBAC5B,KAAK,EAAE,IAAI;gBACX,YAAY,EAAE,MAAM;gBACpB,MAAM,EAAE,aAAa;gBACrB,mBAAmB,EAAE,OAAO,EAAE,QAAe;gBAC7C,YAAY,EAAE,OAAO,EAAE,MAAM;aAC9B,CAAC,CAAC;YACH,OAAO,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAU,CAAC;QACpC,CAAC;QACD,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO;YAC1B,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC;YAC1C,MAAM,MAAM,CAAC,OAAO,CAAC;gBACnB,KAAK,EAAE,IAAI;gBACX,YAAY,EAAE,MAAM;gBACpB,mBAAmB,EAAE,OAAO,EAAE,QAAe;gBAC7C,YAAY,EAAE,OAAO,EAAE,MAAM;aAC9B,CAAC,CAAC;QACL,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,KAAK,EAAE,OAAO;YAC9B,IAAI,YAAY,GAAG,QAAQ,CAAC;YAC5B,IAAI,OAAO,EAAE,KAAK,KAAK,KAAK;gBAAE,YAAY,GAAG,EAAE,CAAC;YAChD,MAAM,CAAC,GAAG,GAAG,CAAA,iBAAiB,KAAK,GAAG,GAAG,CAAC,GAAG,CAAC,YAAY,CAAC,IAAI,KAAK,IAAI,GAAG,CAAC,KAAK,EAAE,EAAE,CAAC;YACtF,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;YACvC,OAAO,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,cAAc,CAAC,KAAK,EAAE,CAAC,CAAC,CAAQ,CAAC;QACzD,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG;YACnB,MAAM,KAAK,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAA,GAAG,GAAG,CAAC,UAAU,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,MAAM,CAAQ,EAAE,CAAC,CAAC;YAC7G,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;gBAAE,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;YACnF,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,CAAA,SAAS,GAAG,CAAC,IAAI,CAAC,KAAK,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;YAClF,OAAO,IAAI,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC;QACzB,CAAC;QACD,MAAM,CAAC,KAAK;YACV,OAAO;gBACL,KAAK,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI;oBACrB,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;oBAC3B,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;wBAAE,OAAO;oBAC9B,MAAM,EAAE,CAAC,SAAS,CAChB,KAAK,CAAC,IAAI,EACV,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,KAAK,EAAE,CAA4B,EAAE,IAAI,CAAC,CAAC,CACzE,CAAC;gBACJ,CAAC;aACF,CAAC;QACJ,CAAC;QACD,KAAK,CAAC,SAAS,CAAC,SAAS,EAAE,IAAI,EAAE,OAAO;YACtC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO;YAC9B,MAAM,MAAM,CAAC,MAAM,CAAC;gBAClB,KAAK,EAAE,SAAS;gBAChB,MAAM,EAAE,IAAI;gBACZ,MAAM,EAAE,aAAa;gBACrB,mBAAmB,EAAE,OAAO,EAAE,QAAe;gBAC7C,YAAY,EAAE,OAAO,EAAE,MAAM;aAC9B,CAAC,CAAC;QACL,CAAC;QACD,KAAK,CAAC,cAAc;YAClB,MAAM,KAAK,GAAG,YAAY,CAAC,EAAE,GAAG,EAAE,MAAM,CAAC,QAAQ,EAAE,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,SAAS,EAAE,eAAe,EAAE,MAAM,CAAC,cAAc,IAAI,MAAM,EAAE,CAAC,CAAC;YAClJ,IAAI,CAAC;gBACH,MAAM,KAAK,CAAC,OAAO,CAAC,EAAE,KAAK,EAAE,mCAAmC,QAAQ,IAAI,EAAE,CAAC,CAAC;YAClF,CAAC;oBAAS,CAAC;gBACT,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;YACtB,CAAC;QACH,CAAC;QACD,KAAK,CAAC,IAAI;YACR,MAAM,CAAC,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;YAC9B,OAAO,CAAC,CAAC,OAAO,CAAC;QACnB,CAAC;QACD,KAAK,CAAC,KAAK;YACT,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;QACvB,CAAC;KACF,CAAC;IACF,OAAO,EAAE,CAAC;AACZ,CAAC"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
export { createDb } from "./client.js";
|
|
2
|
+
export type { Db, DatabaseConfig, QueryOptions } from "./client.js";
|
|
3
|
+
export { sql, renderSql, literal, isSqlFragment } from "./sql.js";
|
|
4
|
+
export type { SqlFragment, SqlRaw, SqlValue, RenderedSql } from "./sql.js";
|
|
5
|
+
export { BatchWriter } from "./batch.js";
|
|
6
|
+
export { serializeRow, deserializeRow } from "./serialize.js";
|
|
7
|
+
export type { RowMeta } from "./serialize.js";
|
|
8
|
+
export { migrate, schemaDdl, collectSchema } from "./migrate.js";
|
|
9
|
+
export type { MigrateOptions, SchemaModule } from "./migrate.js";
|
|
10
|
+
export * as internalTables from "./internal.js";
|
|
11
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/db/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AACvC,YAAY,EAAE,EAAE,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AACpE,OAAO,EAAE,GAAG,EAAE,SAAS,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAClE,YAAY,EAAE,WAAW,EAAE,MAAM,EAAE,QAAQ,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AAC3E,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAC9D,YAAY,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AACjE,YAAY,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACjE,OAAO,KAAK,cAAc,MAAM,eAAe,CAAC"}
|
package/dist/db/index.js
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
export { createDb } from "./client.js";
|
|
2
|
+
export { sql, renderSql, literal, isSqlFragment } from "./sql.js";
|
|
3
|
+
export { BatchWriter } from "./batch.js";
|
|
4
|
+
export { serializeRow, deserializeRow } from "./serialize.js";
|
|
5
|
+
export { migrate, schemaDdl, collectSchema } from "./migrate.js";
|
|
6
|
+
export * as internalTables from "./internal.js";
|
|
7
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/db/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAEvC,OAAO,EAAE,GAAG,EAAE,SAAS,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAElE,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAE9D,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAEjE,OAAO,KAAK,cAAc,MAAM,eAAe,CAAC"}
|