@notuslabs/yail 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +226 -0
- package/bin/yail.js +7 -0
- package/dist/addresses/registry.d.ts +57 -0
- package/dist/addresses/registry.d.ts.map +1 -0
- package/dist/addresses/registry.js +131 -0
- package/dist/addresses/registry.js.map +1 -0
- package/dist/cli/index.d.ts +2 -0
- package/dist/cli/index.d.ts.map +1 -0
- package/dist/cli/index.js +204 -0
- package/dist/cli/index.js.map +1 -0
- package/dist/config/address.d.ts +32 -0
- package/dist/config/address.d.ts.map +1 -0
- package/dist/config/address.js +31 -0
- package/dist/config/address.js.map +1 -0
- package/dist/config/chain-ref.d.ts +5 -0
- package/dist/config/chain-ref.d.ts.map +1 -0
- package/dist/config/chain-ref.js +13 -0
- package/dist/config/chain-ref.js.map +1 -0
- package/dist/config/create-config.d.ts +9 -0
- package/dist/config/create-config.d.ts.map +1 -0
- package/dist/config/create-config.js +86 -0
- package/dist/config/create-config.js.map +1 -0
- package/dist/config/index.d.ts +5 -0
- package/dist/config/index.d.ts.map +1 -0
- package/dist/config/index.js +3 -0
- package/dist/config/index.js.map +1 -0
- package/dist/config/types.d.ts +113 -0
- package/dist/config/types.d.ts.map +1 -0
- package/dist/config/types.js +2 -0
- package/dist/config/types.js.map +1 -0
- package/dist/db/batch.d.ts +33 -0
- package/dist/db/batch.d.ts.map +1 -0
- package/dist/db/batch.js +71 -0
- package/dist/db/batch.js.map +1 -0
- package/dist/db/client.d.ts +50 -0
- package/dist/db/client.d.ts.map +1 -0
- package/dist/db/client.js +104 -0
- package/dist/db/client.js.map +1 -0
- package/dist/db/index.d.ts +11 -0
- package/dist/db/index.d.ts.map +1 -0
- package/dist/db/index.js +7 -0
- package/dist/db/index.js.map +1 -0
- package/dist/db/internal.d.ts +2332 -0
- package/dist/db/internal.d.ts.map +1 -0
- package/dist/db/internal.js +75 -0
- package/dist/db/internal.js.map +1 -0
- package/dist/db/migrate.d.ts +22 -0
- package/dist/db/migrate.d.ts.map +1 -0
- package/dist/db/migrate.js +72 -0
- package/dist/db/migrate.js.map +1 -0
- package/dist/db/serialize.d.ts +14 -0
- package/dist/db/serialize.d.ts.map +1 -0
- package/dist/db/serialize.js +133 -0
- package/dist/db/serialize.js.map +1 -0
- package/dist/db/sql.d.ts +44 -0
- package/dist/db/sql.d.ts.map +1 -0
- package/dist/db/sql.js +122 -0
- package/dist/db/sql.js.map +1 -0
- package/dist/http/client.d.ts +40 -0
- package/dist/http/client.d.ts.map +1 -0
- package/dist/http/client.js +128 -0
- package/dist/http/client.js.map +1 -0
- package/dist/index.d.ts +21 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +15 -0
- package/dist/index.js.map +1 -0
- package/dist/indexer/context.d.ts +84 -0
- package/dist/indexer/context.d.ts.map +1 -0
- package/dist/indexer/context.js +45 -0
- package/dist/indexer/context.js.map +1 -0
- package/dist/indexer/events.d.ts +87 -0
- package/dist/indexer/events.d.ts.map +1 -0
- package/dist/indexer/events.js +88 -0
- package/dist/indexer/events.js.map +1 -0
- package/dist/indexer/indexer.d.ts +114 -0
- package/dist/indexer/indexer.d.ts.map +1 -0
- package/dist/indexer/indexer.js +237 -0
- package/dist/indexer/indexer.js.map +1 -0
- package/dist/indexer/jobs.d.ts +61 -0
- package/dist/indexer/jobs.d.ts.map +1 -0
- package/dist/indexer/jobs.js +170 -0
- package/dist/indexer/jobs.js.map +1 -0
- package/dist/indexer/plan.d.ts +73 -0
- package/dist/indexer/plan.d.ts.map +1 -0
- package/dist/indexer/plan.js +238 -0
- package/dist/indexer/plan.js.map +1 -0
- package/dist/indexer/runtime.d.ts +100 -0
- package/dist/indexer/runtime.d.ts.map +1 -0
- package/dist/indexer/runtime.js +572 -0
- package/dist/indexer/runtime.js.map +1 -0
- package/dist/observability/logger.d.ts +23 -0
- package/dist/observability/logger.d.ts.map +1 -0
- package/dist/observability/logger.js +72 -0
- package/dist/observability/logger.js.map +1 -0
- package/dist/observability/metrics.d.ts +61 -0
- package/dist/observability/metrics.d.ts.map +1 -0
- package/dist/observability/metrics.js +131 -0
- package/dist/observability/metrics.js.map +1 -0
- package/dist/observability/otel.d.ts +28 -0
- package/dist/observability/otel.d.ts.map +1 -0
- package/dist/observability/otel.js +49 -0
- package/dist/observability/otel.js.map +1 -0
- package/dist/rpc/cached-client.d.ts +33 -0
- package/dist/rpc/cached-client.d.ts.map +1 -0
- package/dist/rpc/cached-client.js +50 -0
- package/dist/rpc/cached-client.js.map +1 -0
- package/dist/schema/column.d.ts +61 -0
- package/dist/schema/column.d.ts.map +1 -0
- package/dist/schema/column.js +76 -0
- package/dist/schema/column.js.map +1 -0
- package/dist/schema/ddl.d.ts +9 -0
- package/dist/schema/ddl.d.ts.map +1 -0
- package/dist/schema/ddl.js +58 -0
- package/dist/schema/ddl.js.map +1 -0
- package/dist/schema/index.d.ts +8 -0
- package/dist/schema/index.d.ts.map +1 -0
- package/dist/schema/index.js +5 -0
- package/dist/schema/index.js.map +1 -0
- package/dist/schema/table.d.ts +69 -0
- package/dist/schema/table.d.ts.map +1 -0
- package/dist/schema/table.js +35 -0
- package/dist/schema/table.js.map +1 -0
- package/dist/schema/view.d.ts +19 -0
- package/dist/schema/view.d.ts.map +1 -0
- package/dist/schema/view.js +16 -0
- package/dist/schema/view.js.map +1 -0
- package/dist/server/api.d.ts +11 -0
- package/dist/server/api.d.ts.map +1 -0
- package/dist/server/api.js +71 -0
- package/dist/server/api.js.map +1 -0
- package/dist/sources/cached.d.ts +27 -0
- package/dist/sources/cached.d.ts.map +1 -0
- package/dist/sources/cached.js +111 -0
- package/dist/sources/cached.js.map +1 -0
- package/dist/sources/esplora.d.ts +22 -0
- package/dist/sources/esplora.d.ts.map +1 -0
- package/dist/sources/esplora.js +138 -0
- package/dist/sources/esplora.js.map +1 -0
- package/dist/sources/fixture.d.ts +51 -0
- package/dist/sources/fixture.d.ts.map +1 -0
- package/dist/sources/fixture.js +102 -0
- package/dist/sources/fixture.js.map +1 -0
- package/dist/sources/hypersync.d.ts +19 -0
- package/dist/sources/hypersync.d.ts.map +1 -0
- package/dist/sources/hypersync.js +169 -0
- package/dist/sources/hypersync.js.map +1 -0
- package/dist/sources/index.d.ts +13 -0
- package/dist/sources/index.d.ts.map +1 -0
- package/dist/sources/index.js +7 -0
- package/dist/sources/index.js.map +1 -0
- package/dist/sources/rpc.d.ts +21 -0
- package/dist/sources/rpc.d.ts.map +1 -0
- package/dist/sources/rpc.js +187 -0
- package/dist/sources/rpc.js.map +1 -0
- package/dist/sources/types.d.ts +137 -0
- package/dist/sources/types.d.ts.map +1 -0
- package/dist/sources/types.js +17 -0
- package/dist/sources/types.js.map +1 -0
- package/dist/testing/index.d.ts +20 -0
- package/dist/testing/index.d.ts.map +1 -0
- package/dist/testing/index.js +44 -0
- package/dist/testing/index.js.map +1 -0
- package/dist/util.d.ts +9 -0
- package/dist/util.d.ts.map +1 -0
- package/dist/util.js +25 -0
- package/dist/util.js.map +1 -0
- package/package.json +100 -0
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
import { createLogger, initLogger, log as evlog } from "evlog";
|
|
2
|
+
import { createMemoryDrain, readMemoryLogs } from "evlog/memory";
|
|
3
|
+
import { createOTLPDrain } from "evlog/otlp";
|
|
4
|
+
import { createClickHouseDrain } from "evlog/clickhouse";
|
|
5
|
+
import { Metrics } from "./metrics.js";
|
|
6
|
+
import { decodeOr } from "../util.js";
|
|
7
|
+
let initialized = false;
|
|
8
|
+
const LOGS_TABLE_DDL = `CREATE TABLE IF NOT EXISTS {db}.\`_yail_logs\` (
|
|
9
|
+
timestamp DateTime64(3),
|
|
10
|
+
level LowCardinality(String),
|
|
11
|
+
service LowCardinality(String),
|
|
12
|
+
environment LowCardinality(String),
|
|
13
|
+
request_id String,
|
|
14
|
+
trace_id String,
|
|
15
|
+
span_id String,
|
|
16
|
+
method LowCardinality(String),
|
|
17
|
+
path String,
|
|
18
|
+
status Nullable(UInt16),
|
|
19
|
+
duration String,
|
|
20
|
+
duration_ms UInt32,
|
|
21
|
+
error_name String,
|
|
22
|
+
error_message String,
|
|
23
|
+
data String CODEC(ZSTD(3))
|
|
24
|
+
) ENGINE = MergeTree PARTITION BY toYYYYMM(timestamp) ORDER BY (service, timestamp) TTL toDateTime(timestamp) + INTERVAL 30 DAY`;
|
|
25
|
+
export async function initObservability(config = {}, deps = {}) {
|
|
26
|
+
const serviceName = config.serviceName ?? "yail";
|
|
27
|
+
const drains = [createMemoryDrain({ store: "yail", maxEvents: 2000 })];
|
|
28
|
+
const otlpEndpoint = config.otlpEndpoint ?? process.env.OTEL_EXPORTER_OTLP_ENDPOINT ?? process.env.OTLP_ENDPOINT;
|
|
29
|
+
if (otlpEndpoint) {
|
|
30
|
+
drains.push(createOTLPDrain({ endpoint: otlpEndpoint, serviceName, headers: config.otlpHeaders, recordShape: "compact" }));
|
|
31
|
+
}
|
|
32
|
+
if (config.logsToClickHouse && deps.db && deps.dbConfig) {
|
|
33
|
+
await deps.db.command(LOGS_TABLE_DDL.replace("{db}", `\`${deps.db.database}\``));
|
|
34
|
+
const u = new URL(deps.dbConfig.url);
|
|
35
|
+
const username = deps.dbConfig.username ?? decodeOr(u.username, "default");
|
|
36
|
+
const password = deps.dbConfig.password ?? decodeOr(u.password, "");
|
|
37
|
+
u.username = "";
|
|
38
|
+
u.password = "";
|
|
39
|
+
u.pathname = "/";
|
|
40
|
+
drains.push(createClickHouseDrain({ endpoint: u.toString().replace(/\/$/, ""), database: deps.db.database, table: "_yail_logs", username, password }));
|
|
41
|
+
}
|
|
42
|
+
const drain = async (ctx) => {
|
|
43
|
+
await Promise.allSettled(drains.map((d) => d(ctx)));
|
|
44
|
+
};
|
|
45
|
+
if (!initialized) {
|
|
46
|
+
initLogger({
|
|
47
|
+
env: { service: serviceName, environment: process.env.NODE_ENV ?? "development" },
|
|
48
|
+
pretty: config.pretty ?? process.env.NODE_ENV !== "production",
|
|
49
|
+
minLevel: config.logLevel,
|
|
50
|
+
// YAIL_QUIET=1 is set by one-shot CLI commands whose stdout must stay machine-readable.
|
|
51
|
+
silent: process.env.YAIL_QUIET === "1",
|
|
52
|
+
drain,
|
|
53
|
+
_suppressDrainWarning: true,
|
|
54
|
+
});
|
|
55
|
+
initialized = true;
|
|
56
|
+
}
|
|
57
|
+
const metrics = new Metrics(serviceName);
|
|
58
|
+
return {
|
|
59
|
+
config: { ...config, serviceName, batchEvents: config.batchEvents ?? true },
|
|
60
|
+
log: evlog,
|
|
61
|
+
metrics,
|
|
62
|
+
wide(context) {
|
|
63
|
+
return createLogger(context);
|
|
64
|
+
},
|
|
65
|
+
recentEvents(limit = 100) {
|
|
66
|
+
return readMemoryLogs({ store: "yail", limit });
|
|
67
|
+
},
|
|
68
|
+
};
|
|
69
|
+
}
|
|
70
|
+
/** Structured error with the evlog fields (`why`, `fix`) so alerts explain themselves. */
|
|
71
|
+
export { createError } from "evlog";
|
|
72
|
+
//# sourceMappingURL=logger.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"logger.js","sourceRoot":"","sources":["../../src/observability/logger.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,GAAG,IAAI,KAAK,EAAuD,MAAM,OAAO,CAAC;AACpH,OAAO,EAAE,iBAAiB,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AACjE,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAC7C,OAAO,EAAE,qBAAqB,EAAE,MAAM,kBAAkB,CAAC;AAGzD,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAetC,IAAI,WAAW,GAAG,KAAK,CAAC;AAExB,MAAM,cAAc,GAAG;;;;;;;;;;;;;;;;gIAgByG,CAAC;AAEjI,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,MAAM,GAAwB,EAAE,EAAE,IAAI,GAA2C,EAAE;IACzH,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,IAAI,MAAM,CAAC;IACjD,MAAM,MAAM,GAAc,CAAC,iBAAiB,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IAClF,MAAM,YAAY,GAAG,MAAM,CAAC,YAAY,IAAI,OAAO,CAAC,GAAG,CAAC,2BAA2B,IAAI,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC;IACjH,IAAI,YAAY,EAAE,CAAC;QACjB,MAAM,CAAC,IAAI,CAAC,eAAe,CAAC,EAAE,QAAQ,EAAE,YAAY,EAAE,WAAW,EAAE,OAAO,EAAE,MAAM,CAAC,WAAW,EAAE,WAAW,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC;IAC7H,CAAC;IACD,IAAI,MAAM,CAAC,gBAAgB,IAAI,IAAI,CAAC,EAAE,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;QACxD,MAAM,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,cAAc,CAAC,OAAO,CAAC,MAAM,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAC;QACjF,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;QACrC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,IAAI,QAAQ,CAAC,CAAC,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;QAC3E,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,IAAI,QAAQ,CAAC,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;QACpE,CAAC,CAAC,QAAQ,GAAG,EAAE,CAAC;QAChB,CAAC,CAAC,QAAQ,GAAG,EAAE,CAAC;QAChB,CAAC,CAAC,QAAQ,GAAG,GAAG,CAAC;QACjB,MAAM,CAAC,IAAI,CAAC,qBAAqB,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,EAAE,CAAC,QAAQ,EAAE,KAAK,EAAE,YAAY,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC;IACzJ,CAAC;IACD,MAAM,KAAK,GAAG,KAAK,EAAE,GAAiB,EAAE,EAAE;QACxC,MAAM,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IACtD,CAAC,CAAC;IACF,IAAI,CAAC,WAAW,EAAE,CAAC;QACjB,UAAU,CAAC;YACT,GAAG,EAAE,EAAE,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,OAAO,CAAC,GAAG,CAAC,QAAQ,IAAI,aAAa,EAAE;YACjF,MAAM,EAAE,MAAM,CAAC,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,QAAQ,KAAK,YAAY;YAC9D,QAAQ,EAAE,MAAM,CAAC,QAAQ;YACzB,wFAAwF;YACxF,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,UAAU,KAAK,GAAG;YACtC,KAAK;YACL,qBAAqB,EAAE,IAAI;SAC5B,CAAC,CAAC;QACH,WAAW,GAAG,IAAI,CAAC;IACrB,CAAC;IACD,MAAM,OAAO,GAAG,IAAI,OAAO,CAAC,WAAW,CAAC,CAAC;IACzC,OAAO;QACL,MAAM,EAAE,EAAE,GAAG,MAAM,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,CAAC,WAAW,IAAI,IAAI,EAAE;QAC3E,GAAG,EAAE,KAAK;QACV,OAAO;QACP,IAAI,CAAC,OAAO;YACV,OAAO,YAAY,CAAC,OAAO,CAAC,CAAC;QAC/B,CAAC;QACD,YAAY,CAAC,KAAK,GAAG,GAAG;YACtB,OAAO,cAAc,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;QAClD,CAAC;KACF,CAAC;AACJ,CAAC;AAED,0FAA0F;AAC1F,OAAO,EAAE,WAAW,EAAE,MAAM,OAAO,CAAC"}
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import { type Meter, type Tracer, type Attributes } from "@opentelemetry/api";
|
|
2
|
+
/**
|
|
3
|
+
* OpenTelemetry instruments + an in-process snapshot. The OTEL API is a no-op
|
|
4
|
+
* until an SDK is registered (see `yail/otel`), the snapshot always works and
|
|
5
|
+
* feeds `GET /status`.
|
|
6
|
+
*/
|
|
7
|
+
export interface ChainStats {
|
|
8
|
+
cursor: number;
|
|
9
|
+
head: number;
|
|
10
|
+
finalized: number;
|
|
11
|
+
lagBlocks: number;
|
|
12
|
+
blocksIndexed: number;
|
|
13
|
+
eventsProcessed: number;
|
|
14
|
+
batches: number;
|
|
15
|
+
errors: number;
|
|
16
|
+
lastBatchMs: number;
|
|
17
|
+
lastBatchAt?: string;
|
|
18
|
+
caughtUp: boolean;
|
|
19
|
+
}
|
|
20
|
+
export declare class Metrics {
|
|
21
|
+
readonly meter: Meter;
|
|
22
|
+
readonly tracer: Tracer;
|
|
23
|
+
readonly chains: Map<string, ChainStats>;
|
|
24
|
+
readonly counters: {
|
|
25
|
+
sourceRequests: number;
|
|
26
|
+
cacheHits: number;
|
|
27
|
+
cacheMisses: number;
|
|
28
|
+
httpCacheHits: number;
|
|
29
|
+
httpCacheMisses: number;
|
|
30
|
+
rpcCacheHits: number;
|
|
31
|
+
rpcCacheMisses: number;
|
|
32
|
+
rowsFlushed: number;
|
|
33
|
+
jobsDone: number;
|
|
34
|
+
jobsFailed: number;
|
|
35
|
+
};
|
|
36
|
+
private blocksIndexed;
|
|
37
|
+
private eventsProcessed;
|
|
38
|
+
private batchDuration;
|
|
39
|
+
private handlerDuration;
|
|
40
|
+
private flushRows;
|
|
41
|
+
private flushDuration;
|
|
42
|
+
private errors;
|
|
43
|
+
private cacheEvents;
|
|
44
|
+
private jobEvents;
|
|
45
|
+
private sourceDuration;
|
|
46
|
+
constructor(serviceName?: string);
|
|
47
|
+
chain(name: string): ChainStats;
|
|
48
|
+
setProgress(chain: string, cursor: number, head: number, finalized: number): void;
|
|
49
|
+
recordBatch(chain: string, blocks: number, events: number, ms: number, sourceMs: number): void;
|
|
50
|
+
recordEvent(chain: string, name: string, ms: number): void;
|
|
51
|
+
recordFlush(table: string, rows: number, ms: number): void;
|
|
52
|
+
recordError(chain: string | undefined, stage: string): void;
|
|
53
|
+
recordCache(cache: "source" | "http" | "rpc", hit: boolean): void;
|
|
54
|
+
recordJob(kind: string, outcome: "done" | "failed"): void;
|
|
55
|
+
snapshot(): {
|
|
56
|
+
chains: Record<string, ChainStats>;
|
|
57
|
+
counters: Metrics["counters"];
|
|
58
|
+
};
|
|
59
|
+
span<T>(name: string, attributes: Attributes, fn: () => Promise<T>): Promise<T>;
|
|
60
|
+
}
|
|
61
|
+
//# sourceMappingURL=metrics.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"metrics.d.ts","sourceRoot":"","sources":["../../src/observability/metrics.ts"],"names":[],"mappings":"AAAA,OAAO,EAAiC,KAAK,KAAK,EAAE,KAAK,MAAM,EAAE,KAAK,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAE7G;;;;GAIG;AACH,MAAM,WAAW,UAAU;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,EAAE,MAAM,CAAC;IACxB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,EAAE,OAAO,CAAC;CACnB;AASD,qBAAa,OAAO;IAClB,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,MAAM,0BAAiC;IAChD,QAAQ,CAAC,QAAQ;QAAK,cAAc;QAAK,SAAS;QAAK,WAAW;QAAK,aAAa;QAAK,eAAe;QAAK,YAAY;QAAK,cAAc;QAAK,WAAW;QAAK,QAAQ;QAAK,UAAU;MAAM;IAE9L,OAAO,CAAC,aAAa,CAAC;IACtB,OAAO,CAAC,eAAe,CAAC;IACxB,OAAO,CAAC,aAAa,CAAC;IACtB,OAAO,CAAC,eAAe,CAAC;IACxB,OAAO,CAAC,SAAS,CAAC;IAClB,OAAO,CAAC,aAAa,CAAC;IACtB,OAAO,CAAC,MAAM,CAAC;IACf,OAAO,CAAC,WAAW,CAAC;IACpB,OAAO,CAAC,SAAS,CAAC;IAClB,OAAO,CAAC,cAAc,CAAC;IAEvB,YAAY,WAAW,SAAS,EA2B/B;IAED,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,UAAU,CAO9B;IAED,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,IAAI,CAOhF;IAED,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,IAAI,CAU7F;IAED,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,GAAG,IAAI,CAGzD;IAED,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,GAAG,IAAI,CAIzD;IAED,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAG1D;IAED,WAAW,CAAC,KAAK,EAAE,QAAQ,GAAG,MAAM,GAAG,KAAK,EAAE,GAAG,EAAE,OAAO,GAAG,IAAI,CAUhE;IAED,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,QAAQ,GAAG,IAAI,CAIxD;IAED,QAAQ,IAAI;QAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;QAAC,QAAQ,EAAE,OAAO,CAAC,UAAU,CAAC,CAAA;KAAE,CAEhF;IAED,IAAI,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,UAAU,EAAE,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAY9E;CACF"}
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
import { metrics as otelMetrics, trace } from "@opentelemetry/api";
|
|
2
|
+
const CACHE_COUNTERS = {
|
|
3
|
+
source: ["cacheHits", "cacheMisses"],
|
|
4
|
+
http: ["httpCacheHits", "httpCacheMisses"],
|
|
5
|
+
rpc: ["rpcCacheHits", "rpcCacheMisses"],
|
|
6
|
+
};
|
|
7
|
+
export class Metrics {
|
|
8
|
+
meter;
|
|
9
|
+
tracer;
|
|
10
|
+
chains = new Map();
|
|
11
|
+
counters = { sourceRequests: 0, cacheHits: 0, cacheMisses: 0, httpCacheHits: 0, httpCacheMisses: 0, rpcCacheHits: 0, rpcCacheMisses: 0, rowsFlushed: 0, jobsDone: 0, jobsFailed: 0 };
|
|
12
|
+
blocksIndexed;
|
|
13
|
+
eventsProcessed;
|
|
14
|
+
batchDuration;
|
|
15
|
+
handlerDuration;
|
|
16
|
+
flushRows;
|
|
17
|
+
flushDuration;
|
|
18
|
+
errors;
|
|
19
|
+
cacheEvents;
|
|
20
|
+
jobEvents;
|
|
21
|
+
sourceDuration;
|
|
22
|
+
constructor(serviceName = "yail") {
|
|
23
|
+
this.meter = otelMetrics.getMeter(serviceName);
|
|
24
|
+
this.tracer = trace.getTracer(serviceName);
|
|
25
|
+
const m = this.meter;
|
|
26
|
+
this.blocksIndexed = m.createCounter("yail.blocks.indexed", { description: "Blocks indexed", unit: "{block}" });
|
|
27
|
+
this.eventsProcessed = m.createCounter("yail.events.processed", { description: "Events dispatched to handlers", unit: "{event}" });
|
|
28
|
+
this.batchDuration = m.createHistogram("yail.batch.duration", { description: "Source batch end-to-end processing time", unit: "ms" });
|
|
29
|
+
this.handlerDuration = m.createHistogram("yail.handler.duration", { description: "Handler execution time", unit: "ms" });
|
|
30
|
+
this.flushRows = m.createCounter("yail.flush.rows", { description: "Rows written to ClickHouse", unit: "{row}" });
|
|
31
|
+
this.flushDuration = m.createHistogram("yail.flush.duration", { description: "ClickHouse insert time per table", unit: "ms" });
|
|
32
|
+
this.errors = m.createCounter("yail.errors", { description: "Errors by stage" });
|
|
33
|
+
this.cacheEvents = m.createCounter("yail.cache.events", { description: "Cache hits/misses by cache" });
|
|
34
|
+
this.jobEvents = m.createCounter("yail.jobs", { description: "Backfill / reindex jobs by outcome" });
|
|
35
|
+
this.sourceDuration = m.createHistogram("yail.source.duration", { description: "Time waiting on the data source per batch", unit: "ms" });
|
|
36
|
+
m.createObservableGauge("yail.chain.head", { description: "Source head block" }).addCallback((r) => {
|
|
37
|
+
for (const [chain, s] of this.chains)
|
|
38
|
+
r.observe(s.head, { chain });
|
|
39
|
+
});
|
|
40
|
+
m.createObservableGauge("yail.chain.cursor", { description: "Next block to index" }).addCallback((r) => {
|
|
41
|
+
for (const [chain, s] of this.chains)
|
|
42
|
+
r.observe(s.cursor, { chain });
|
|
43
|
+
});
|
|
44
|
+
m.createObservableGauge("yail.chain.lag_blocks", { description: "finalized head - cursor. Alert when this grows." }).addCallback((r) => {
|
|
45
|
+
for (const [chain, s] of this.chains)
|
|
46
|
+
r.observe(s.lagBlocks, { chain });
|
|
47
|
+
});
|
|
48
|
+
m.createObservableGauge("yail.chain.caught_up", { description: "1 when the chain is at the finalized head" }).addCallback((r) => {
|
|
49
|
+
for (const [chain, s] of this.chains)
|
|
50
|
+
r.observe(Number(s.caughtUp), { chain });
|
|
51
|
+
});
|
|
52
|
+
}
|
|
53
|
+
chain(name) {
|
|
54
|
+
let s = this.chains.get(name);
|
|
55
|
+
if (!s) {
|
|
56
|
+
s = { cursor: 0, head: 0, finalized: 0, lagBlocks: 0, blocksIndexed: 0, eventsProcessed: 0, batches: 0, errors: 0, lastBatchMs: 0, caughtUp: false };
|
|
57
|
+
this.chains.set(name, s);
|
|
58
|
+
}
|
|
59
|
+
return s;
|
|
60
|
+
}
|
|
61
|
+
setProgress(chain, cursor, head, finalized) {
|
|
62
|
+
const s = this.chain(chain);
|
|
63
|
+
s.cursor = cursor;
|
|
64
|
+
s.head = head;
|
|
65
|
+
s.finalized = finalized;
|
|
66
|
+
s.lagBlocks = Math.max(0, finalized + 1 - cursor);
|
|
67
|
+
s.caughtUp = cursor > finalized;
|
|
68
|
+
}
|
|
69
|
+
recordBatch(chain, blocks, events, ms, sourceMs) {
|
|
70
|
+
const s = this.chain(chain);
|
|
71
|
+
s.blocksIndexed += blocks;
|
|
72
|
+
s.eventsProcessed += events;
|
|
73
|
+
s.batches++;
|
|
74
|
+
s.lastBatchMs = ms;
|
|
75
|
+
s.lastBatchAt = new Date().toISOString();
|
|
76
|
+
this.blocksIndexed.add(blocks, { chain });
|
|
77
|
+
this.batchDuration.record(ms, { chain });
|
|
78
|
+
this.sourceDuration.record(sourceMs, { chain });
|
|
79
|
+
}
|
|
80
|
+
recordEvent(chain, name, ms) {
|
|
81
|
+
this.eventsProcessed.add(1, { chain, event: name });
|
|
82
|
+
this.handlerDuration.record(ms, { chain, event: name });
|
|
83
|
+
}
|
|
84
|
+
recordFlush(table, rows, ms) {
|
|
85
|
+
this.counters.rowsFlushed += rows;
|
|
86
|
+
this.flushRows.add(rows, { table });
|
|
87
|
+
this.flushDuration.record(ms, { table });
|
|
88
|
+
}
|
|
89
|
+
recordError(chain, stage) {
|
|
90
|
+
if (chain)
|
|
91
|
+
this.chain(chain).errors++;
|
|
92
|
+
this.errors.add(1, { chain: chain ?? "", stage });
|
|
93
|
+
}
|
|
94
|
+
recordCache(cache, hit) {
|
|
95
|
+
const [hitCounter, missCounter] = CACHE_COUNTERS[cache];
|
|
96
|
+
let result = "miss";
|
|
97
|
+
let counter = missCounter;
|
|
98
|
+
if (hit) {
|
|
99
|
+
result = "hit";
|
|
100
|
+
counter = hitCounter;
|
|
101
|
+
}
|
|
102
|
+
this.counters[counter]++;
|
|
103
|
+
this.cacheEvents.add(1, { cache, result });
|
|
104
|
+
}
|
|
105
|
+
recordJob(kind, outcome) {
|
|
106
|
+
if (outcome === "done")
|
|
107
|
+
this.counters.jobsDone++;
|
|
108
|
+
else
|
|
109
|
+
this.counters.jobsFailed++;
|
|
110
|
+
this.jobEvents.add(1, { kind, outcome });
|
|
111
|
+
}
|
|
112
|
+
snapshot() {
|
|
113
|
+
return { chains: Object.fromEntries(this.chains), counters: { ...this.counters } };
|
|
114
|
+
}
|
|
115
|
+
span(name, attributes, fn) {
|
|
116
|
+
return this.tracer.startActiveSpan(name, { attributes }, async (span) => {
|
|
117
|
+
try {
|
|
118
|
+
return await fn();
|
|
119
|
+
}
|
|
120
|
+
catch (err) {
|
|
121
|
+
span.recordException(err);
|
|
122
|
+
span.setStatus({ code: 2, message: err.message });
|
|
123
|
+
throw err;
|
|
124
|
+
}
|
|
125
|
+
finally {
|
|
126
|
+
span.end();
|
|
127
|
+
}
|
|
128
|
+
});
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
//# sourceMappingURL=metrics.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"metrics.js","sourceRoot":"","sources":["../../src/observability/metrics.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,IAAI,WAAW,EAAE,KAAK,EAA4C,MAAM,oBAAoB,CAAC;AAsB7G,MAAM,cAAc,GAAkE;IACpF,MAAM,EAAE,CAAC,WAAW,EAAE,aAAa,CAAC;IACpC,IAAI,EAAE,CAAC,eAAe,EAAE,iBAAiB,CAAC;IAC1C,GAAG,EAAE,CAAC,cAAc,EAAE,gBAAgB,CAAC;CACxC,CAAC;AAEF,MAAM,OAAO,OAAO;IACT,KAAK,CAAQ;IACb,MAAM,CAAS;IACf,MAAM,GAAG,IAAI,GAAG,EAAsB,CAAC;IACvC,QAAQ,GAAG,EAAE,cAAc,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE,eAAe,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,cAAc,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,CAAC;IAEtL,aAAa,CAAC;IACd,eAAe,CAAC;IAChB,aAAa,CAAC;IACd,eAAe,CAAC;IAChB,SAAS,CAAC;IACV,aAAa,CAAC;IACd,MAAM,CAAC;IACP,WAAW,CAAC;IACZ,SAAS,CAAC;IACV,cAAc,CAAC;IAEvB,YAAY,WAAW,GAAG,MAAM;QAC9B,IAAI,CAAC,KAAK,GAAG,WAAW,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;QAC/C,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;QAC3C,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC;QACrB,IAAI,CAAC,aAAa,GAAG,CAAC,CAAC,aAAa,CAAC,qBAAqB,EAAE,EAAE,WAAW,EAAE,gBAAgB,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC;QAChH,IAAI,CAAC,eAAe,GAAG,CAAC,CAAC,aAAa,CAAC,uBAAuB,EAAE,EAAE,WAAW,EAAE,+BAA+B,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC;QACnI,IAAI,CAAC,aAAa,GAAG,CAAC,CAAC,eAAe,CAAC,qBAAqB,EAAE,EAAE,WAAW,EAAE,yCAAyC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QACtI,IAAI,CAAC,eAAe,GAAG,CAAC,CAAC,eAAe,CAAC,uBAAuB,EAAE,EAAE,WAAW,EAAE,wBAAwB,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QACzH,IAAI,CAAC,SAAS,GAAG,CAAC,CAAC,aAAa,CAAC,iBAAiB,EAAE,EAAE,WAAW,EAAE,4BAA4B,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;QAClH,IAAI,CAAC,aAAa,GAAG,CAAC,CAAC,eAAe,CAAC,qBAAqB,EAAE,EAAE,WAAW,EAAE,kCAAkC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QAC/H,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,aAAa,CAAC,aAAa,EAAE,EAAE,WAAW,EAAE,iBAAiB,EAAE,CAAC,CAAC;QACjF,IAAI,CAAC,WAAW,GAAG,CAAC,CAAC,aAAa,CAAC,mBAAmB,EAAE,EAAE,WAAW,EAAE,4BAA4B,EAAE,CAAC,CAAC;QACvG,IAAI,CAAC,SAAS,GAAG,CAAC,CAAC,aAAa,CAAC,WAAW,EAAE,EAAE,WAAW,EAAE,oCAAoC,EAAE,CAAC,CAAC;QACrG,IAAI,CAAC,cAAc,GAAG,CAAC,CAAC,eAAe,CAAC,sBAAsB,EAAE,EAAE,WAAW,EAAE,2CAA2C,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QAE1I,CAAC,CAAC,qBAAqB,CAAC,iBAAiB,EAAE,EAAE,WAAW,EAAE,mBAAmB,EAAE,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,EAAE;YACjG,KAAK,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM;gBAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;QACrE,CAAC,CAAC,CAAC;QACH,CAAC,CAAC,qBAAqB,CAAC,mBAAmB,EAAE,EAAE,WAAW,EAAE,qBAAqB,EAAE,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,EAAE;YACrG,KAAK,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM;gBAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;QACvE,CAAC,CAAC,CAAC;QACH,CAAC,CAAC,qBAAqB,CAAC,uBAAuB,EAAE,EAAE,WAAW,EAAE,iDAAiD,EAAE,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,EAAE;YACrI,KAAK,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM;gBAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;QAC1E,CAAC,CAAC,CAAC;QACH,CAAC,CAAC,qBAAqB,CAAC,sBAAsB,EAAE,EAAE,WAAW,EAAE,2CAA2C,EAAE,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,EAAE;YAC9H,KAAK,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM;gBAAE,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;QACjF,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,IAAY;QAChB,IAAI,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAC9B,IAAI,CAAC,CAAC,EAAE,CAAC;YACP,CAAC,GAAG,EAAE,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE,eAAe,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;YACrJ,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAC3B,CAAC;QACD,OAAO,CAAC,CAAC;IACX,CAAC;IAED,WAAW,CAAC,KAAa,EAAE,MAAc,EAAE,IAAY,EAAE,SAAiB;QACxE,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAC5B,CAAC,CAAC,MAAM,GAAG,MAAM,CAAC;QAClB,CAAC,CAAC,IAAI,GAAG,IAAI,CAAC;QACd,CAAC,CAAC,SAAS,GAAG,SAAS,CAAC;QACxB,CAAC,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC;QAClD,CAAC,CAAC,QAAQ,GAAG,MAAM,GAAG,SAAS,CAAC;IAClC,CAAC;IAED,WAAW,CAAC,KAAa,EAAE,MAAc,EAAE,MAAc,EAAE,EAAU,EAAE,QAAgB;QACrF,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAC5B,CAAC,CAAC,aAAa,IAAI,MAAM,CAAC;QAC1B,CAAC,CAAC,eAAe,IAAI,MAAM,CAAC;QAC5B,CAAC,CAAC,OAAO,EAAE,CAAC;QACZ,CAAC,CAAC,WAAW,GAAG,EAAE,CAAC;QACnB,CAAC,CAAC,WAAW,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QACzC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,MAAM,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;QAC1C,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;QACzC,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;IAClD,CAAC;IAED,WAAW,CAAC,KAAa,EAAE,IAAY,EAAE,EAAU;QACjD,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACpD,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IAC1D,CAAC;IAED,WAAW,CAAC,KAAa,EAAE,IAAY,EAAE,EAAU;QACjD,IAAI,CAAC,QAAQ,CAAC,WAAW,IAAI,IAAI,CAAC;QAClC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;QACpC,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;IAC3C,CAAC;IAED,WAAW,CAAC,KAAyB,EAAE,KAAa;QAClD,IAAI,KAAK;YAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,EAAE,CAAC;QACtC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,KAAK,IAAI,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;IACpD,CAAC;IAED,WAAW,CAAC,KAAgC,EAAE,GAAY;QACxD,MAAM,CAAC,UAAU,EAAE,WAAW,CAAC,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;QACxD,IAAI,MAAM,GAAG,MAAM,CAAC;QACpB,IAAI,OAAO,GAAG,WAAW,CAAC;QAC1B,IAAI,GAAG,EAAE,CAAC;YACR,MAAM,GAAG,KAAK,CAAC;YACf,OAAO,GAAG,UAAU,CAAC;QACvB,CAAC;QACD,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;QACzB,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;IAC7C,CAAC;IAED,SAAS,CAAC,IAAY,EAAE,OAA0B;QAChD,IAAI,OAAO,KAAK,MAAM;YAAE,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC;;YAC5C,IAAI,CAAC,QAAQ,CAAC,UAAU,EAAE,CAAC;QAChC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;IAC3C,CAAC;IAED,QAAQ;QACN,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,EAAE,GAAG,IAAI,CAAC,QAAQ,EAAE,EAAE,CAAC;IACrF,CAAC;IAED,IAAI,CAAI,IAAY,EAAE,UAAsB,EAAE,EAAoB;QAChE,OAAO,IAAI,CAAC,MAAM,CAAC,eAAe,CAAC,IAAI,EAAE,EAAE,UAAU,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE;YACtE,IAAI,CAAC;gBACH,OAAO,MAAM,EAAE,EAAE,CAAC;YACpB,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,IAAI,CAAC,eAAe,CAAC,GAAY,CAAC,CAAC;gBACnC,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,OAAO,EAAG,GAAa,CAAC,OAAO,EAAE,CAAC,CAAC;gBAC7D,MAAM,GAAG,CAAC;YACZ,CAAC;oBAAS,CAAC;gBACT,IAAI,CAAC,GAAG,EAAE,CAAC;YACb,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;CACF"}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import { type MetricReader } from "@opentelemetry/sdk-metrics";
|
|
2
|
+
export interface TelemetryOptions {
|
|
3
|
+
serviceName?: string;
|
|
4
|
+
serviceVersion?: string;
|
|
5
|
+
/** OTLP HTTP base endpoint, e.g. http://localhost:4318. Falls back to OTEL_EXPORTER_OTLP_ENDPOINT. */
|
|
6
|
+
endpoint?: string;
|
|
7
|
+
headers?: Record<string, string>;
|
|
8
|
+
/** Metric export interval in ms. Default 15000. */
|
|
9
|
+
metricsIntervalMs?: number;
|
|
10
|
+
/** Extra metric readers (e.g. a Prometheus exporter). */
|
|
11
|
+
metricReaders?: MetricReader[];
|
|
12
|
+
/** Extra resource attributes (deployment.environment, ...). */
|
|
13
|
+
resourceAttributes?: Record<string, string>;
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* Register the OpenTelemetry SDK so `yail` metrics and traces are exported over
|
|
17
|
+
* OTLP/HTTP. Call this once at process start (before `indexer.start()`).
|
|
18
|
+
* Logs go through evlog's OTLP drain, configured by `observability.otlpEndpoint`.
|
|
19
|
+
*
|
|
20
|
+
* @example
|
|
21
|
+
* import { startTelemetry } from "@notuslabs/yail/otel";
|
|
22
|
+
* const telemetry = startTelemetry({ serviceName: "wallet-ledger", endpoint: "http://otel-collector:4318" });
|
|
23
|
+
* process.on("SIGTERM", () => telemetry.shutdown());
|
|
24
|
+
*/
|
|
25
|
+
export declare function startTelemetry(options?: TelemetryOptions): {
|
|
26
|
+
shutdown(): Promise<void>;
|
|
27
|
+
};
|
|
28
|
+
//# sourceMappingURL=otel.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"otel.d.ts","sourceRoot":"","sources":["../../src/observability/otel.ts"],"names":[],"mappings":"AAGA,OAAO,EAAiC,KAAK,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAI9F,MAAM,WAAW,gBAAgB;IAC/B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,sGAAsG;IACtG,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,mDAAmD;IACnD,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,yDAAyD;IACzD,aAAa,CAAC,EAAE,YAAY,EAAE,CAAC;IAC/B,+DAA+D;IAC/D,kBAAkB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAC7C;AAED;;;;;;;;;GASG;AACH,wBAAgB,cAAc,CAAC,OAAO,GAAE,gBAAqB,GAAG;IAAE,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC,CAAA;CAAE,CAmB5F"}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import { NodeSDK } from "@opentelemetry/sdk-node";
|
|
2
|
+
import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";
|
|
3
|
+
import { OTLPMetricExporter } from "@opentelemetry/exporter-metrics-otlp-http";
|
|
4
|
+
import { PeriodicExportingMetricReader } from "@opentelemetry/sdk-metrics";
|
|
5
|
+
import { resourceFromAttributes } from "@opentelemetry/resources";
|
|
6
|
+
import { ATTR_SERVICE_NAME, ATTR_SERVICE_VERSION } from "@opentelemetry/semantic-conventions";
|
|
7
|
+
/**
|
|
8
|
+
* Register the OpenTelemetry SDK so `yail` metrics and traces are exported over
|
|
9
|
+
* OTLP/HTTP. Call this once at process start (before `indexer.start()`).
|
|
10
|
+
* Logs go through evlog's OTLP drain, configured by `observability.otlpEndpoint`.
|
|
11
|
+
*
|
|
12
|
+
* @example
|
|
13
|
+
* import { startTelemetry } from "@notuslabs/yail/otel";
|
|
14
|
+
* const telemetry = startTelemetry({ serviceName: "wallet-ledger", endpoint: "http://otel-collector:4318" });
|
|
15
|
+
* process.on("SIGTERM", () => telemetry.shutdown());
|
|
16
|
+
*/
|
|
17
|
+
export function startTelemetry(options = {}) {
|
|
18
|
+
const endpoint = (options.endpoint ?? process.env.OTEL_EXPORTER_OTLP_ENDPOINT ?? "http://localhost:4318").replace(/\/$/, "");
|
|
19
|
+
const serviceName = options.serviceName ?? process.env.OTEL_SERVICE_NAME ?? "yail";
|
|
20
|
+
const headers = options.headers ?? parseHeaders(process.env.OTEL_EXPORTER_OTLP_HEADERS);
|
|
21
|
+
const attributes = { [ATTR_SERVICE_NAME]: serviceName, ...(options.resourceAttributes ?? {}) };
|
|
22
|
+
if (options.serviceVersion)
|
|
23
|
+
attributes[ATTR_SERVICE_VERSION] = options.serviceVersion;
|
|
24
|
+
const sdk = new NodeSDK({
|
|
25
|
+
resource: resourceFromAttributes(attributes),
|
|
26
|
+
traceExporter: new OTLPTraceExporter({ url: `${endpoint}/v1/traces`, headers }),
|
|
27
|
+
metricReaders: [
|
|
28
|
+
new PeriodicExportingMetricReader({
|
|
29
|
+
exporter: new OTLPMetricExporter({ url: `${endpoint}/v1/metrics`, headers }),
|
|
30
|
+
exportIntervalMillis: options.metricsIntervalMs ?? 15_000,
|
|
31
|
+
}),
|
|
32
|
+
...(options.metricReaders ?? []),
|
|
33
|
+
],
|
|
34
|
+
});
|
|
35
|
+
sdk.start();
|
|
36
|
+
return { shutdown: () => sdk.shutdown() };
|
|
37
|
+
}
|
|
38
|
+
function parseHeaders(raw) {
|
|
39
|
+
if (!raw)
|
|
40
|
+
return undefined;
|
|
41
|
+
const out = {};
|
|
42
|
+
for (const pair of raw.split(",")) {
|
|
43
|
+
const [k, ...rest] = pair.split("=");
|
|
44
|
+
if (k && rest.length)
|
|
45
|
+
out[k.trim()] = decodeURIComponent(rest.join("=").trim());
|
|
46
|
+
}
|
|
47
|
+
return out;
|
|
48
|
+
}
|
|
49
|
+
//# sourceMappingURL=otel.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"otel.js","sourceRoot":"","sources":["../../src/observability/otel.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAClD,OAAO,EAAE,iBAAiB,EAAE,MAAM,yCAAyC,CAAC;AAC5E,OAAO,EAAE,kBAAkB,EAAE,MAAM,2CAA2C,CAAC;AAC/E,OAAO,EAAE,6BAA6B,EAAqB,MAAM,4BAA4B,CAAC;AAC9F,OAAO,EAAE,sBAAsB,EAAE,MAAM,0BAA0B,CAAC;AAClE,OAAO,EAAE,iBAAiB,EAAE,oBAAoB,EAAE,MAAM,qCAAqC,CAAC;AAgB9F;;;;;;;;;GASG;AACH,MAAM,UAAU,cAAc,CAAC,OAAO,GAAqB,EAAE;IAC3D,MAAM,QAAQ,GAAG,CAAC,OAAO,CAAC,QAAQ,IAAI,OAAO,CAAC,GAAG,CAAC,2BAA2B,IAAI,uBAAuB,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IAC7H,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,MAAM,CAAC;IACnF,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,0BAA0B,CAAC,CAAC;IACxF,MAAM,UAAU,GAA2B,EAAE,CAAC,iBAAiB,CAAC,EAAE,WAAW,EAAE,GAAG,CAAC,OAAO,CAAC,kBAAkB,IAAI,EAAE,CAAC,EAAE,CAAC;IACvH,IAAI,OAAO,CAAC,cAAc;QAAE,UAAU,CAAC,oBAAoB,CAAC,GAAG,OAAO,CAAC,cAAc,CAAC;IACtF,MAAM,GAAG,GAAG,IAAI,OAAO,CAAC;QACtB,QAAQ,EAAE,sBAAsB,CAAC,UAAU,CAAC;QAC5C,aAAa,EAAE,IAAI,iBAAiB,CAAC,EAAE,GAAG,EAAE,GAAG,QAAQ,YAAY,EAAE,OAAO,EAAE,CAAC;QAC/E,aAAa,EAAE;YACb,IAAI,6BAA6B,CAAC;gBAChC,QAAQ,EAAE,IAAI,kBAAkB,CAAC,EAAE,GAAG,EAAE,GAAG,QAAQ,aAAa,EAAE,OAAO,EAAE,CAAC;gBAC5E,oBAAoB,EAAE,OAAO,CAAC,iBAAiB,IAAI,MAAM;aAC1D,CAAC;YACF,GAAG,CAAC,OAAO,CAAC,aAAa,IAAI,EAAE,CAAC;SACjC;KACF,CAAC,CAAC;IACH,GAAG,CAAC,KAAK,EAAE,CAAC;IACZ,OAAO,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,CAAC;AAC5C,CAAC;AAED,SAAS,YAAY,CAAC,GAAY;IAChC,IAAI,CAAC,GAAG;QAAE,OAAO,SAAS,CAAC;IAC3B,MAAM,GAAG,GAA2B,EAAE,CAAC;IACvC,KAAK,MAAM,IAAI,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;QAClC,MAAM,CAAC,CAAC,EAAE,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACrC,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM;YAAE,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,GAAG,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAClF,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC"}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import { decodeFunctionResult, type Abi, type ContractFunctionArgs, type ContractFunctionName, type PublicClient } from "viem";
|
|
2
|
+
import type { Db } from "../db/client.js";
|
|
3
|
+
export interface ReadContractParams<abi extends Abi, fn extends ContractFunctionName<abi, "pure" | "view">> {
|
|
4
|
+
address: string;
|
|
5
|
+
abi: abi;
|
|
6
|
+
functionName: fn;
|
|
7
|
+
args?: ContractFunctionArgs<abi, "pure" | "view", fn>;
|
|
8
|
+
/** Block to read at. Defaults to the block of the event being processed. */
|
|
9
|
+
blockNumber?: number | bigint;
|
|
10
|
+
/** Bypass the cache for this call. */
|
|
11
|
+
noCache?: boolean;
|
|
12
|
+
}
|
|
13
|
+
export interface CachedClient {
|
|
14
|
+
/** Underlying viem client for anything else. */
|
|
15
|
+
readonly raw: PublicClient;
|
|
16
|
+
/** `eth_call` cached by (chain, block, to, calldata) in ClickHouse. Typed like viem's readContract. */
|
|
17
|
+
readContract<abi extends Abi, fn extends ContractFunctionName<abi, "pure" | "view">>(params: ReadContractParams<abi, fn>): Promise<ReturnType<typeof decodeFunctionResult<abi, fn>>>;
|
|
18
|
+
/** Contract reads with the same block/cache semantics, run in parallel. */
|
|
19
|
+
multicall<T extends ReadonlyArray<ReadContractParams<any, any>>>(calls: T): Promise<{
|
|
20
|
+
[K in keyof T]: unknown;
|
|
21
|
+
}>;
|
|
22
|
+
}
|
|
23
|
+
export interface CachedClientOptions {
|
|
24
|
+
chain: string;
|
|
25
|
+
url: string;
|
|
26
|
+
db?: Db;
|
|
27
|
+
persist?: boolean;
|
|
28
|
+
onCache?: (hit: boolean) => void;
|
|
29
|
+
/** Provided by the runtime: the block currently being processed. */
|
|
30
|
+
currentBlock?: () => number | undefined;
|
|
31
|
+
}
|
|
32
|
+
export declare function createCachedClient(options: CachedClientOptions): CachedClient;
|
|
33
|
+
//# sourceMappingURL=cached-client.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cached-client.d.ts","sourceRoot":"","sources":["../../src/rpc/cached-client.ts"],"names":[],"mappings":"AAAA,OAAO,EAAsB,oBAAoB,EAA4B,KAAK,GAAG,EAAE,KAAK,oBAAoB,EAAE,KAAK,oBAAoB,EAAE,KAAK,YAAY,EAAE,MAAM,MAAM,CAAC;AAC7K,OAAO,KAAK,EAAE,EAAE,EAAE,MAAM,iBAAiB,CAAC;AAI1C,MAAM,WAAW,kBAAkB,CAAC,GAAG,SAAS,GAAG,EAAE,EAAE,SAAS,oBAAoB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAC;IACxG,OAAO,EAAE,MAAM,CAAC;IAChB,GAAG,EAAE,GAAG,CAAC;IACT,YAAY,EAAE,EAAE,CAAC;IACjB,IAAI,CAAC,EAAE,oBAAoB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,EAAE,EAAE,CAAC,CAAC;IACtD,4EAA4E;IAC5E,WAAW,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IAC9B,sCAAsC;IACtC,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAED,MAAM,WAAW,YAAY;IAC3B,gDAAgD;IAChD,QAAQ,CAAC,GAAG,EAAE,YAAY,CAAC;IAC3B,uGAAuG;IACvG,YAAY,CAAC,GAAG,SAAS,GAAG,EAAE,EAAE,SAAS,oBAAoB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAC,EACjF,MAAM,EAAE,kBAAkB,CAAC,GAAG,EAAE,EAAE,CAAC,GAClC,OAAO,CAAC,UAAU,CAAC,OAAO,oBAAoB,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;IAC7D,2EAA2E;IAC3E,SAAS,CAAC,CAAC,SAAS,aAAa,CAAC,kBAAkB,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,GAAG,OAAO,CAAC;SAAG,CAAC,IAAI,MAAM,CAAC,GAAG,OAAO;KAAE,CAAC,CAAC;CAClH;AAED,MAAM,WAAW,mBAAmB;IAClC,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,EAAE,CAAC,EAAE,EAAE,CAAC;IACR,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,OAAO,CAAC,EAAE,CAAC,GAAG,EAAE,OAAO,KAAK,IAAI,CAAC;IACjC,oEAAoE;IACpE,YAAY,CAAC,EAAE,MAAM,MAAM,GAAG,SAAS,CAAC;CACzC;AAED,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,mBAAmB,GAAG,YAAY,CA6C7E"}
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import { createPublicClient, decodeFunctionResult, encodeFunctionData, http } from "viem";
|
|
2
|
+
import { rpcCache } from "../db/internal.js";
|
|
3
|
+
import { sql } from "../db/sql.js";
|
|
4
|
+
export function createCachedClient(options) {
|
|
5
|
+
const raw = createPublicClient({ transport: http(options.url, { batch: true, retryCount: 4 }) });
|
|
6
|
+
const persist = options.persist !== false && options.db !== undefined;
|
|
7
|
+
const memory = new Map();
|
|
8
|
+
async function readContract(params) {
|
|
9
|
+
const data = encodeFunctionData({ abi: params.abi, functionName: params.functionName, args: params.args });
|
|
10
|
+
const blockNumber = params.blockNumber ?? options.currentBlock?.();
|
|
11
|
+
const to = params.address.toLowerCase();
|
|
12
|
+
let block;
|
|
13
|
+
if (blockNumber !== undefined)
|
|
14
|
+
block = Number(blockNumber);
|
|
15
|
+
let key;
|
|
16
|
+
if (block !== undefined)
|
|
17
|
+
key = `${block}:${to}:${data}`;
|
|
18
|
+
let result;
|
|
19
|
+
if (key && !params.noCache) {
|
|
20
|
+
result = memory.get(key);
|
|
21
|
+
if (result === undefined && persist) {
|
|
22
|
+
const rows = await options.db.query(sql `SELECT result FROM ${rpcCache} FINAL WHERE chain = ${options.chain} AND block = ${block} AND to = ${to} AND data = ${data} LIMIT 1`);
|
|
23
|
+
result = rows[0]?.result;
|
|
24
|
+
if (result !== undefined)
|
|
25
|
+
memory.set(key, result);
|
|
26
|
+
}
|
|
27
|
+
options.onCache?.(result !== undefined);
|
|
28
|
+
}
|
|
29
|
+
if (result === undefined) {
|
|
30
|
+
let blockTag;
|
|
31
|
+
if (block !== undefined)
|
|
32
|
+
blockTag = BigInt(block);
|
|
33
|
+
const res = await raw.call({ to: to, data, blockNumber: blockTag });
|
|
34
|
+
result = res.data ?? "0x";
|
|
35
|
+
if (key && !params.noCache) {
|
|
36
|
+
memory.set(key, result);
|
|
37
|
+
if (persist) {
|
|
38
|
+
await options.db.insert(rpcCache).values({ chain: options.chain, block: BigInt(block), to, data, result, createdAt: new Date() });
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
return decodeFunctionResult({ abi: params.abi, functionName: params.functionName, data: result });
|
|
43
|
+
}
|
|
44
|
+
return {
|
|
45
|
+
raw,
|
|
46
|
+
readContract: readContract,
|
|
47
|
+
multicall: (calls) => Promise.all(calls.map((c) => readContract(c))),
|
|
48
|
+
};
|
|
49
|
+
}
|
|
50
|
+
//# sourceMappingURL=cached-client.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cached-client.js","sourceRoot":"","sources":["../../src/rpc/cached-client.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,kBAAkB,EAAE,IAAI,EAAqF,MAAM,MAAM,CAAC;AAE7K,OAAO,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAC7C,OAAO,EAAE,GAAG,EAAE,MAAM,cAAc,CAAC;AAkCnC,MAAM,UAAU,kBAAkB,CAAC,OAA4B;IAC7D,MAAM,GAAG,GAAG,kBAAkB,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;IACjG,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,KAAK,KAAK,IAAI,OAAO,CAAC,EAAE,KAAK,SAAS,CAAC;IACtE,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IAEzC,KAAK,UAAU,YAAY,CAAC,MAAoC;QAC9D,MAAM,IAAI,GAAG,kBAAkB,CAAC,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,YAAY,EAAE,MAAM,CAAC,YAAY,EAAE,IAAI,EAAE,MAAM,CAAC,IAAW,EAAE,CAAC,CAAC;QAClH,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,IAAI,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC;QACnE,MAAM,EAAE,GAAG,MAAM,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;QACxC,IAAI,KAAyB,CAAC;QAC9B,IAAI,WAAW,KAAK,SAAS;YAAE,KAAK,GAAG,MAAM,CAAC,WAAW,CAAC,CAAC;QAC3D,IAAI,GAAuB,CAAC;QAC5B,IAAI,KAAK,KAAK,SAAS;YAAE,GAAG,GAAG,GAAG,KAAK,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC;QACxD,IAAI,MAA0B,CAAC;QAC/B,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YAC3B,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACzB,IAAI,MAAM,KAAK,SAAS,IAAI,OAAO,EAAE,CAAC;gBACpC,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,EAAG,CAAC,KAAK,CAClC,GAAG,CAAA,sBAAsB,QAAQ,wBAAwB,OAAO,CAAC,KAAK,gBAAgB,KAAM,aAAa,EAAE,eAAe,IAAI,UAAU,CACzI,CAAC;gBACF,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC;gBACzB,IAAI,MAAM,KAAK,SAAS;oBAAE,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;YACpD,CAAC;YACD,OAAO,CAAC,OAAO,EAAE,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC;QAC1C,CAAC;QACD,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YACzB,IAAI,QAA4B,CAAC;YACjC,IAAI,KAAK,KAAK,SAAS;gBAAE,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;YAClD,MAAM,GAAG,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,EAAmB,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC,CAAC;YACrF,MAAM,GAAG,GAAG,CAAC,IAAI,IAAI,IAAI,CAAC;YAC1B,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;gBAC3B,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;gBACxB,IAAI,OAAO,EAAE,CAAC;oBACZ,MAAM,OAAO,CAAC,EAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC,KAAM,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,IAAI,EAAE,EAAE,CAAC,CAAC;gBACtI,CAAC;YACH,CAAC;QACH,CAAC;QACD,OAAO,oBAAoB,CAAC,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,YAAY,EAAE,MAAM,CAAC,YAAY,EAAE,IAAI,EAAE,MAAuB,EAAE,CAAC,CAAC;IACrH,CAAC;IAED,OAAO;QACL,GAAG;QACH,YAAY,EAAE,YAA4C;QAC1D,SAAS,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAQ;KAC5E,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Column builders for the YAIL schema DSL.
|
|
3
|
+
*
|
|
4
|
+
* Each builder carries two things: the ClickHouse type it maps to, and a
|
|
5
|
+
* phantom TypeScript type used to infer row types (`InferRow<typeof table>`).
|
|
6
|
+
* Serialization to/from ClickHouse's JSONEachRow format is driven by `kind`.
|
|
7
|
+
*/
|
|
8
|
+
export type ColumnKind = "string" | "address" | "hash" | "bool" | "int" | "bigint" | "float" | "date" | "enum" | "json" | "array";
|
|
9
|
+
export interface Column<T = unknown, Nullable extends boolean = boolean, HasDefault extends boolean = boolean> {
|
|
10
|
+
readonly __type?: T;
|
|
11
|
+
readonly kind: ColumnKind;
|
|
12
|
+
readonly chType: string;
|
|
13
|
+
readonly isNullable: Nullable;
|
|
14
|
+
readonly hasDefault: HasDefault;
|
|
15
|
+
readonly defaultSql?: string;
|
|
16
|
+
readonly codecSql?: string;
|
|
17
|
+
readonly isLowCardinality: boolean;
|
|
18
|
+
readonly items?: Column;
|
|
19
|
+
}
|
|
20
|
+
type Builder<T, N extends boolean, D extends boolean> = Column<T, N, D> & {
|
|
21
|
+
nullable(): Builder<T, true, D>;
|
|
22
|
+
/** Column default (raw ClickHouse expression). Rows may omit the column on insert. */
|
|
23
|
+
default(sql: string): Builder<T, N, true>;
|
|
24
|
+
codec(codec: string): Builder<T, N, D>;
|
|
25
|
+
lowCardinality(): Builder<T, N, D>;
|
|
26
|
+
};
|
|
27
|
+
export declare const t: {
|
|
28
|
+
readonly string: () => Builder<string, false, false>;
|
|
29
|
+
/** Lowercased hex address (EVM) or plain address string (Bitcoin). */
|
|
30
|
+
readonly address: () => Builder<string, false, false>;
|
|
31
|
+
/** 0x-prefixed hex hash. */
|
|
32
|
+
readonly hash: () => Builder<string, false, false>;
|
|
33
|
+
readonly bool: () => Builder<boolean, false, false>;
|
|
34
|
+
readonly uint8: () => Builder<number, false, false>;
|
|
35
|
+
readonly uint16: () => Builder<number, false, false>;
|
|
36
|
+
readonly uint32: () => Builder<number, false, false>;
|
|
37
|
+
readonly int8: () => Builder<number, false, false>;
|
|
38
|
+
readonly int16: () => Builder<number, false, false>;
|
|
39
|
+
readonly int32: () => Builder<number, false, false>;
|
|
40
|
+
readonly uint64: () => Builder<bigint, false, false>;
|
|
41
|
+
readonly int64: () => Builder<bigint, false, false>;
|
|
42
|
+
readonly uint128: () => Builder<bigint, false, false>;
|
|
43
|
+
readonly int128: () => Builder<bigint, false, false>;
|
|
44
|
+
readonly uint256: () => Builder<bigint, false, false>;
|
|
45
|
+
readonly int256: () => Builder<bigint, false, false>;
|
|
46
|
+
/** Alias for int256: the safe default for any EVM amount, signed so deltas fit. */
|
|
47
|
+
readonly bigint: () => Builder<bigint, false, false>;
|
|
48
|
+
readonly float32: () => Builder<number, false, false>;
|
|
49
|
+
readonly float64: () => Builder<number, false, false>;
|
|
50
|
+
/** Decimal with given precision/scale, exposed as string to avoid float loss. */
|
|
51
|
+
readonly decimal: (precision: number, scale: number) => Builder<string, false, false>;
|
|
52
|
+
readonly date: () => Builder<Date, false, false>;
|
|
53
|
+
readonly dateTime: () => Builder<Date, false, false>;
|
|
54
|
+
readonly dateTime64: (precision?: number) => Builder<Date, false, false>;
|
|
55
|
+
readonly enum: <const V extends readonly string[]>(values: V) => Builder<V[number], false, false>;
|
|
56
|
+
readonly json: <T = unknown>() => Builder<T, false, false>;
|
|
57
|
+
readonly array: <C extends Column<any, boolean, boolean>>(items: C) => Builder<NonNullable<C["__type"]>[], false, false>;
|
|
58
|
+
};
|
|
59
|
+
export declare function columnTypeSql(col: Column): string;
|
|
60
|
+
export {};
|
|
61
|
+
//# sourceMappingURL=column.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"column.d.ts","sourceRoot":"","sources":["../../src/schema/column.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,MAAM,MAAM,UAAU,GAClB,QAAQ,GACR,SAAS,GACT,MAAM,GACN,MAAM,GACN,KAAK,GACL,QAAQ,GACR,OAAO,GACP,MAAM,GACN,MAAM,GACN,MAAM,GACN,OAAO,CAAC;AAEZ,MAAM,WAAW,MAAM,CAAC,CAAC,GAAG,OAAO,EAAE,QAAQ,SAAS,OAAO,GAAG,OAAO,EAAE,UAAU,SAAS,OAAO,GAAG,OAAO;IAC3G,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACpB,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;IAC1B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,UAAU,EAAE,QAAQ,CAAC;IAC9B,QAAQ,CAAC,UAAU,EAAE,UAAU,CAAC;IAChC,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,gBAAgB,EAAE,OAAO,CAAC;IACnC,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,KAAK,OAAO,CAAC,CAAC,EAAE,CAAC,SAAS,OAAO,EAAE,CAAC,SAAS,OAAO,IAAI,MAAM,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,GAAG;IACxE,QAAQ,IAAI,OAAO,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IAChC,sFAAsF;IACtF,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,EAAE,CAAC,EAAE,IAAI,CAAC,CAAC;IAC1C,KAAK,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;IACvC,cAAc,IAAI,OAAO,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;CACpC,CAAC;AAgCF,eAAO,MAAM,CAAC;aACZ,MAAM;IACN,sEAAsE;aACtE,OAAO;IACP,4BAA4B;aAC5B,IAAI;aACJ,IAAI;aACJ,KAAK;aACL,MAAM;aACN,MAAM;aACN,IAAI;aACJ,KAAK;aACL,KAAK;aACL,MAAM;aACN,KAAK;aACL,OAAO;aACP,MAAM;aACN,OAAO;aACP,MAAM;IACN,mFAAmF;aACnF,MAAM;aACN,OAAO;aACP,OAAO;IACP,iFAAiF;aACjF,OAAO,cAAc,MAAM,SAAS,MAAM;aAC1C,IAAI;aACJ,QAAQ;aACR,UAAU;aACV,IAAI,GAAG,KAAK,CAAC,CAAC,SAAS,SAAS,MAAM,EAAE,UAAU,CAAC;aAEnD,IAAI,GAAG,CAAC,GAAG,OAAO;aAClB,KAAK,GAAG,CAAC,SAAS,MAAM,CAAC,GAAG,EAAE,OAAO,EAAE,OAAO,CAAC,SAAS,CAAC;CAEjD,CAAC;AAEX,wBAAgB,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAKjD"}
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Column builders for the YAIL schema DSL.
|
|
3
|
+
*
|
|
4
|
+
* Each builder carries two things: the ClickHouse type it maps to, and a
|
|
5
|
+
* phantom TypeScript type used to infer row types (`InferRow<typeof table>`).
|
|
6
|
+
* Serialization to/from ClickHouse's JSONEachRow format is driven by `kind`.
|
|
7
|
+
*/
|
|
8
|
+
function make(kind, chType, extra = {}) {
|
|
9
|
+
const col = {
|
|
10
|
+
kind,
|
|
11
|
+
chType,
|
|
12
|
+
isNullable: false,
|
|
13
|
+
hasDefault: false,
|
|
14
|
+
isLowCardinality: false,
|
|
15
|
+
...extra,
|
|
16
|
+
};
|
|
17
|
+
return withMethods(col);
|
|
18
|
+
}
|
|
19
|
+
function withMethods(col) {
|
|
20
|
+
return {
|
|
21
|
+
...col,
|
|
22
|
+
nullable() {
|
|
23
|
+
return withMethods({ ...col, isNullable: true });
|
|
24
|
+
},
|
|
25
|
+
default(sql) {
|
|
26
|
+
return withMethods({ ...col, hasDefault: true, defaultSql: sql });
|
|
27
|
+
},
|
|
28
|
+
codec(codec) {
|
|
29
|
+
return withMethods({ ...col, codecSql: codec });
|
|
30
|
+
},
|
|
31
|
+
lowCardinality() {
|
|
32
|
+
return withMethods({ ...col, isLowCardinality: true });
|
|
33
|
+
},
|
|
34
|
+
};
|
|
35
|
+
}
|
|
36
|
+
export const t = {
|
|
37
|
+
string: () => make("string", "String"),
|
|
38
|
+
/** Lowercased hex address (EVM) or plain address string (Bitcoin). */
|
|
39
|
+
address: () => make("address", "String"),
|
|
40
|
+
/** 0x-prefixed hex hash. */
|
|
41
|
+
hash: () => make("hash", "String"),
|
|
42
|
+
bool: () => make("bool", "Bool"),
|
|
43
|
+
uint8: () => make("int", "UInt8"),
|
|
44
|
+
uint16: () => make("int", "UInt16"),
|
|
45
|
+
uint32: () => make("int", "UInt32"),
|
|
46
|
+
int8: () => make("int", "Int8"),
|
|
47
|
+
int16: () => make("int", "Int16"),
|
|
48
|
+
int32: () => make("int", "Int32"),
|
|
49
|
+
uint64: () => make("bigint", "UInt64"),
|
|
50
|
+
int64: () => make("bigint", "Int64"),
|
|
51
|
+
uint128: () => make("bigint", "UInt128"),
|
|
52
|
+
int128: () => make("bigint", "Int128"),
|
|
53
|
+
uint256: () => make("bigint", "UInt256"),
|
|
54
|
+
int256: () => make("bigint", "Int256"),
|
|
55
|
+
/** Alias for int256: the safe default for any EVM amount, signed so deltas fit. */
|
|
56
|
+
bigint: () => make("bigint", "Int256"),
|
|
57
|
+
float32: () => make("float", "Float32"),
|
|
58
|
+
float64: () => make("float", "Float64"),
|
|
59
|
+
/** Decimal with given precision/scale, exposed as string to avoid float loss. */
|
|
60
|
+
decimal: (precision, scale) => make("string", `Decimal(${precision}, ${scale})`),
|
|
61
|
+
date: () => make("date", "Date"),
|
|
62
|
+
dateTime: () => make("date", "DateTime"),
|
|
63
|
+
dateTime64: (precision = 3) => make("date", `DateTime64(${precision})`),
|
|
64
|
+
enum: (values) => make("enum", `Enum8(${values.map((v, i) => `'${v}' = ${i + 1}`).join(", ")})`),
|
|
65
|
+
json: () => make("json", "String"),
|
|
66
|
+
array: (items) => make("array", `Array(${columnTypeSql(items)})`, { items }),
|
|
67
|
+
};
|
|
68
|
+
export function columnTypeSql(col) {
|
|
69
|
+
let type = col.chType;
|
|
70
|
+
if (col.isLowCardinality)
|
|
71
|
+
type = `LowCardinality(${type})`;
|
|
72
|
+
if (col.isNullable)
|
|
73
|
+
type = `Nullable(${type})`;
|
|
74
|
+
return type;
|
|
75
|
+
}
|
|
76
|
+
//# sourceMappingURL=column.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"column.js","sourceRoot":"","sources":["../../src/schema/column.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAmCH,SAAS,IAAI,CAAI,IAAgB,EAAE,MAAc,EAAE,KAAK,GAAyC,EAAE;IACjG,MAAM,GAAG,GAAgC;QACvC,IAAI;QACJ,MAAM;QACN,UAAU,EAAE,KAAK;QACjB,UAAU,EAAE,KAAK;QACjB,gBAAgB,EAAE,KAAK;QACvB,GAAG,KAAK;KACT,CAAC;IACF,OAAO,WAAW,CAAC,GAAG,CAA6B,CAAC;AACtD,CAAC;AAED,SAAS,WAAW,CAA0C,GAAoB;IAChF,OAAO;QACL,GAAG,GAAG;QACN,QAAQ;YACN,OAAO,WAAW,CAAC,EAAE,GAAG,GAAG,EAAE,UAAU,EAAE,IAAa,EAAE,CAAwB,CAAC;QACnF,CAAC;QACD,OAAO,CAAC,GAAW;YACjB,OAAO,WAAW,CAAC,EAAE,GAAG,GAAG,EAAE,UAAU,EAAE,IAAa,EAAE,UAAU,EAAE,GAAG,EAAE,CAAwB,CAAC;QACpG,CAAC;QACD,KAAK,CAAC,KAAa;YACjB,OAAO,WAAW,CAAC,EAAE,GAAG,GAAG,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;QAClD,CAAC;QACD,cAAc;YACZ,OAAO,WAAW,CAAC,EAAE,GAAG,GAAG,EAAE,gBAAgB,EAAE,IAAI,EAAE,CAAC,CAAC;QACzD,CAAC;KACF,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,MAAM,CAAC,GAAG;IACf,MAAM,EAAE,GAAG,EAAE,CAAC,IAAI,CAAS,QAAQ,EAAE,QAAQ,CAAC;IAC9C,sEAAsE;IACtE,OAAO,EAAE,GAAG,EAAE,CAAC,IAAI,CAAS,SAAS,EAAE,QAAQ,CAAC;IAChD,4BAA4B;IAC5B,IAAI,EAAE,GAAG,EAAE,CAAC,IAAI,CAAS,MAAM,EAAE,QAAQ,CAAC;IAC1C,IAAI,EAAE,GAAG,EAAE,CAAC,IAAI,CAAU,MAAM,EAAE,MAAM,CAAC;IACzC,KAAK,EAAE,GAAG,EAAE,CAAC,IAAI,CAAS,KAAK,EAAE,OAAO,CAAC;IACzC,MAAM,EAAE,GAAG,EAAE,CAAC,IAAI,CAAS,KAAK,EAAE,QAAQ,CAAC;IAC3C,MAAM,EAAE,GAAG,EAAE,CAAC,IAAI,CAAS,KAAK,EAAE,QAAQ,CAAC;IAC3C,IAAI,EAAE,GAAG,EAAE,CAAC,IAAI,CAAS,KAAK,EAAE,MAAM,CAAC;IACvC,KAAK,EAAE,GAAG,EAAE,CAAC,IAAI,CAAS,KAAK,EAAE,OAAO,CAAC;IACzC,KAAK,EAAE,GAAG,EAAE,CAAC,IAAI,CAAS,KAAK,EAAE,OAAO,CAAC;IACzC,MAAM,EAAE,GAAG,EAAE,CAAC,IAAI,CAAS,QAAQ,EAAE,QAAQ,CAAC;IAC9C,KAAK,EAAE,GAAG,EAAE,CAAC,IAAI,CAAS,QAAQ,EAAE,OAAO,CAAC;IAC5C,OAAO,EAAE,GAAG,EAAE,CAAC,IAAI,CAAS,QAAQ,EAAE,SAAS,CAAC;IAChD,MAAM,EAAE,GAAG,EAAE,CAAC,IAAI,CAAS,QAAQ,EAAE,QAAQ,CAAC;IAC9C,OAAO,EAAE,GAAG,EAAE,CAAC,IAAI,CAAS,QAAQ,EAAE,SAAS,CAAC;IAChD,MAAM,EAAE,GAAG,EAAE,CAAC,IAAI,CAAS,QAAQ,EAAE,QAAQ,CAAC;IAC9C,mFAAmF;IACnF,MAAM,EAAE,GAAG,EAAE,CAAC,IAAI,CAAS,QAAQ,EAAE,QAAQ,CAAC;IAC9C,OAAO,EAAE,GAAG,EAAE,CAAC,IAAI,CAAS,OAAO,EAAE,SAAS,CAAC;IAC/C,OAAO,EAAE,GAAG,EAAE,CAAC,IAAI,CAAS,OAAO,EAAE,SAAS,CAAC;IAC/C,iFAAiF;IACjF,OAAO,EAAE,CAAC,SAAiB,EAAE,KAAa,EAAE,EAAE,CAAC,IAAI,CAAS,QAAQ,EAAE,WAAW,SAAS,KAAK,KAAK,GAAG,CAAC;IACxG,IAAI,EAAE,GAAG,EAAE,CAAC,IAAI,CAAO,MAAM,EAAE,MAAM,CAAC;IACtC,QAAQ,EAAE,GAAG,EAAE,CAAC,IAAI,CAAO,MAAM,EAAE,UAAU,CAAC;IAC9C,UAAU,EAAE,CAAC,SAAS,GAAG,CAAC,EAAE,EAAE,CAAC,IAAI,CAAO,MAAM,EAAE,cAAc,SAAS,GAAG,CAAC;IAC7E,IAAI,EAAE,CAAoC,MAAS,EAAE,EAAE,CACrD,IAAI,CAAY,MAAM,EAAE,SAAS,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC;IAC3F,IAAI,EAAE,GAAgB,EAAE,CAAC,IAAI,CAAI,MAAM,EAAE,QAAQ,CAAC;IAClD,KAAK,EAAE,CAA0C,KAAQ,EAAE,EAAE,CAC3D,IAAI,CAAkC,OAAO,EAAE,SAAS,aAAa,CAAC,KAAK,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,CAAC;CACrF,CAAC;AAEX,MAAM,UAAU,aAAa,CAAC,GAAW;IACvC,IAAI,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC;IACtB,IAAI,GAAG,CAAC,gBAAgB;QAAE,IAAI,GAAG,kBAAkB,IAAI,GAAG,CAAC;IAC3D,IAAI,GAAG,CAAC,UAAU;QAAE,IAAI,GAAG,YAAY,IAAI,GAAG,CAAC;IAC/C,OAAO,IAAI,CAAC;AACd,CAAC"}
|