@wide-events/collector 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +6 -0
- package/README.md +20 -0
- package/dist/cli.d.ts +3 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/cli.js +11 -0
- package/dist/cli.js.map +1 -0
- package/dist/config.d.ts +13 -0
- package/dist/config.d.ts.map +1 -0
- package/dist/config.js +29 -0
- package/dist/config.js.map +1 -0
- package/dist/index.d.ts +4 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +4 -0
- package/dist/index.js.map +1 -0
- package/dist/jobs/retention.d.ts +9 -0
- package/dist/jobs/retention.d.ts.map +1 -0
- package/dist/jobs/retention.js +19 -0
- package/dist/jobs/retention.js.map +1 -0
- package/dist/otlp/flatten.d.ts +5 -0
- package/dist/otlp/flatten.d.ts.map +1 -0
- package/dist/otlp/flatten.js +139 -0
- package/dist/otlp/flatten.js.map +1 -0
- package/dist/otlp/types.d.ts +39 -0
- package/dist/otlp/types.d.ts.map +1 -0
- package/dist/otlp/types.js +2 -0
- package/dist/otlp/types.js.map +1 -0
- package/dist/query/build-query.d.ts +8 -0
- package/dist/query/build-query.d.ts.map +1 -0
- package/dist/query/build-query.js +104 -0
- package/dist/query/build-query.js.map +1 -0
- package/dist/routes/columns.d.ts +4 -0
- package/dist/routes/columns.d.ts.map +1 -0
- package/dist/routes/columns.js +6 -0
- package/dist/routes/columns.js.map +1 -0
- package/dist/routes/health.d.ts +3 -0
- package/dist/routes/health.d.ts.map +1 -0
- package/dist/routes/health.js +6 -0
- package/dist/routes/health.js.map +1 -0
- package/dist/routes/query.d.ts +4 -0
- package/dist/routes/query.d.ts.map +1 -0
- package/dist/routes/query.js +12 -0
- package/dist/routes/query.js.map +1 -0
- package/dist/routes/sql.d.ts +4 -0
- package/dist/routes/sql.d.ts.map +1 -0
- package/dist/routes/sql.js +15 -0
- package/dist/routes/sql.js.map +1 -0
- package/dist/routes/trace.d.ts +4 -0
- package/dist/routes/trace.d.ts.map +1 -0
- package/dist/routes/trace.js +11 -0
- package/dist/routes/trace.js.map +1 -0
- package/dist/routes/v1-traces.d.ts +4 -0
- package/dist/routes/v1-traces.d.ts.map +1 -0
- package/dist/routes/v1-traces.js +11 -0
- package/dist/routes/v1-traces.js.map +1 -0
- package/dist/server.d.ts +21 -0
- package/dist/server.d.ts.map +1 -0
- package/dist/server.js +60 -0
- package/dist/server.js.map +1 -0
- package/dist/storage/database.d.ts +12 -0
- package/dist/storage/database.d.ts.map +1 -0
- package/dist/storage/database.js +82 -0
- package/dist/storage/database.js.map +1 -0
- package/dist/storage/schema-registry.d.ts +12 -0
- package/dist/storage/schema-registry.d.ts.map +1 -0
- package/dist/storage/schema-registry.js +59 -0
- package/dist/storage/schema-registry.js.map +1 -0
- package/dist/storage/serialized-executor.d.ts +5 -0
- package/dist/storage/serialized-executor.d.ts.map +1 -0
- package/dist/storage/serialized-executor.js +9 -0
- package/dist/storage/serialized-executor.js.map +1 -0
- package/dist/storage/store.d.ts +19 -0
- package/dist/storage/store.d.ts.map +1 -0
- package/dist/storage/store.js +134 -0
- package/dist/storage/store.js.map +1 -0
- package/package.json +37 -0
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
export function registerTraceQueryRoutes(app, dependencies) {
|
|
2
|
+
app.get("/trace/:id", async (request) => {
|
|
3
|
+
const params = request.params;
|
|
4
|
+
const rows = await dependencies.database.executeRead(`SELECT * FROM events WHERE trace_id = ? ORDER BY ts ASC`, [params.id]);
|
|
5
|
+
return {
|
|
6
|
+
traceId: params.id,
|
|
7
|
+
rows
|
|
8
|
+
};
|
|
9
|
+
});
|
|
10
|
+
}
|
|
11
|
+
//# sourceMappingURL=trace.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"trace.js","sourceRoot":"","sources":["../../src/routes/trace.ts"],"names":[],"mappings":"AAGA,MAAM,UAAU,wBAAwB,CACtC,GAAoB,EACpB,YAAmC;IAEnC,GAAG,CAAC,GAAG,CAAC,YAAY,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;QACtC,MAAM,MAAM,GAAG,OAAO,CAAC,MAAwB,CAAC;QAChD,MAAM,IAAI,GAAG,MAAM,YAAY,CAAC,QAAQ,CAAC,WAAW,CAClD,yDAAyD,EACzD,CAAC,MAAM,CAAC,EAAE,CAAC,CACZ,CAAC;QACF,OAAO;YACL,OAAO,EAAE,MAAM,CAAC,EAAE;YAClB,IAAI;SACL,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"v1-traces.d.ts","sourceRoot":"","sources":["../../src/routes/v1-traces.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAG/C,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,cAAc,CAAC;AAE1D,wBAAgB,mBAAmB,CACjC,GAAG,EAAE,eAAe,EACpB,YAAY,EAAE,qBAAqB,GAClC,IAAI,CAON"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { assertRecord } from "@wide-events/internal";
|
|
2
|
+
import { flattenTraceRequest } from "../otlp/flatten.js";
|
|
3
|
+
export function registerTraceRoutes(app, dependencies) {
|
|
4
|
+
app.post("/v1/traces", async (request, reply) => {
|
|
5
|
+
assertRecord(request.body, "OTLP request body");
|
|
6
|
+
const rows = flattenTraceRequest(request.body);
|
|
7
|
+
await dependencies.store.enqueueRows(rows);
|
|
8
|
+
await reply.code(202).send({ accepted: rows.length });
|
|
9
|
+
});
|
|
10
|
+
}
|
|
11
|
+
//# sourceMappingURL=v1-traces.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"v1-traces.js","sourceRoot":"","sources":["../../src/routes/v1-traces.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACrD,OAAO,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAGzD,MAAM,UAAU,mBAAmB,CACjC,GAAoB,EACpB,YAAmC;IAEnC,GAAG,CAAC,IAAI,CAAC,YAAY,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE;QAC9C,YAAY,CAAC,OAAO,CAAC,IAAI,EAAE,mBAAmB,CAAC,CAAC;QAChD,MAAM,IAAI,GAAG,mBAAmB,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QAC/C,MAAM,YAAY,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;QAC3C,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;IACxD,CAAC,CAAC,CAAC;AACL,CAAC"}
|
package/dist/server.d.ts
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { type FastifyInstance } from "fastify";
|
|
2
|
+
import type { CollectorConfig } from "./config.js";
|
|
3
|
+
import { DuckDbDatabase } from "./storage/database.js";
|
|
4
|
+
import { SchemaRegistry } from "./storage/schema-registry.js";
|
|
5
|
+
import { CollectorStore } from "./storage/store.js";
|
|
6
|
+
import { RetentionJob } from "./jobs/retention.js";
|
|
7
|
+
export interface CollectorDependencies {
|
|
8
|
+
config: CollectorConfig;
|
|
9
|
+
database: DuckDbDatabase;
|
|
10
|
+
schema: SchemaRegistry;
|
|
11
|
+
store: CollectorStore;
|
|
12
|
+
retentionJob: RetentionJob;
|
|
13
|
+
}
|
|
14
|
+
export interface CollectorServer {
|
|
15
|
+
app: FastifyInstance;
|
|
16
|
+
dependencies: CollectorDependencies;
|
|
17
|
+
start(): Promise<void>;
|
|
18
|
+
close(): Promise<void>;
|
|
19
|
+
}
|
|
20
|
+
export declare function createCollectorServer(config?: CollectorConfig): Promise<CollectorServer>;
|
|
21
|
+
//# sourceMappingURL=server.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../src/server.ts"],"names":[],"mappings":"AAAA,OAAgB,EAAE,KAAK,eAAe,EAAE,MAAM,SAAS,CAAC;AACxD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAQnD,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvD,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAC9D,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACpD,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAEnD,MAAM,WAAW,qBAAqB;IACpC,MAAM,EAAE,eAAe,CAAC;IACxB,QAAQ,EAAE,cAAc,CAAC;IACzB,MAAM,EAAE,cAAc,CAAC;IACvB,KAAK,EAAE,cAAc,CAAC;IACtB,YAAY,EAAE,YAAY,CAAC;CAC5B;AAED,MAAM,WAAW,eAAe;IAC9B,GAAG,EAAE,eAAe,CAAC;IACrB,YAAY,EAAE,qBAAqB,CAAC;IACpC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACvB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACxB;AAED,wBAAsB,qBAAqB,CACzC,MAAM,GAAE,eAAuC,GAC9C,OAAO,CAAC,eAAe,CAAC,CAkD1B"}
|
package/dist/server.js
ADDED
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
import Fastify from "fastify";
|
|
2
|
+
import { readCollectorConfig } from "./config.js";
|
|
3
|
+
import { registerColumnRoutes } from "./routes/columns.js";
|
|
4
|
+
import { registerHealthRoute } from "./routes/health.js";
|
|
5
|
+
import { registerQueryRoutes } from "./routes/query.js";
|
|
6
|
+
import { registerSqlRoutes } from "./routes/sql.js";
|
|
7
|
+
import { registerTraceQueryRoutes } from "./routes/trace.js";
|
|
8
|
+
import { registerTraceRoutes } from "./routes/v1-traces.js";
|
|
9
|
+
import { DuckDbDatabase } from "./storage/database.js";
|
|
10
|
+
import { SchemaRegistry } from "./storage/schema-registry.js";
|
|
11
|
+
import { CollectorStore } from "./storage/store.js";
|
|
12
|
+
import { RetentionJob } from "./jobs/retention.js";
|
|
13
|
+
export async function createCollectorServer(config = readCollectorConfig()) {
|
|
14
|
+
const database = await DuckDbDatabase.create(config.duckDbPath);
|
|
15
|
+
const schema = new SchemaRegistry(config.maxColumns);
|
|
16
|
+
await schema.hydrate(database);
|
|
17
|
+
const store = new CollectorStore(database, schema, config);
|
|
18
|
+
const retentionJob = new RetentionJob(store);
|
|
19
|
+
const dependencies = {
|
|
20
|
+
config,
|
|
21
|
+
database,
|
|
22
|
+
schema,
|
|
23
|
+
store,
|
|
24
|
+
retentionJob
|
|
25
|
+
};
|
|
26
|
+
const app = Fastify({
|
|
27
|
+
logger: true
|
|
28
|
+
});
|
|
29
|
+
registerHealthRoute(app);
|
|
30
|
+
registerTraceRoutes(app, dependencies);
|
|
31
|
+
registerQueryRoutes(app, dependencies);
|
|
32
|
+
registerSqlRoutes(app, dependencies);
|
|
33
|
+
registerColumnRoutes(app, dependencies);
|
|
34
|
+
registerTraceQueryRoutes(app, dependencies);
|
|
35
|
+
app.setErrorHandler((error, _request, reply) => {
|
|
36
|
+
app.log.error({ err: error }, "collector request failed");
|
|
37
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
38
|
+
void reply.code(400).send({
|
|
39
|
+
error: message
|
|
40
|
+
});
|
|
41
|
+
});
|
|
42
|
+
return {
|
|
43
|
+
app,
|
|
44
|
+
dependencies,
|
|
45
|
+
async start() {
|
|
46
|
+
retentionJob.start();
|
|
47
|
+
await app.listen({
|
|
48
|
+
port: config.port,
|
|
49
|
+
host: "0.0.0.0"
|
|
50
|
+
});
|
|
51
|
+
},
|
|
52
|
+
async close() {
|
|
53
|
+
retentionJob.stop();
|
|
54
|
+
await store.flush();
|
|
55
|
+
await app.close();
|
|
56
|
+
database.close();
|
|
57
|
+
}
|
|
58
|
+
};
|
|
59
|
+
}
|
|
60
|
+
//# sourceMappingURL=server.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"server.js","sourceRoot":"","sources":["../src/server.ts"],"names":[],"mappings":"AAAA,OAAO,OAAiC,MAAM,SAAS,CAAC;AAExD,OAAO,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAClD,OAAO,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AAC3D,OAAO,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AACzD,OAAO,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAC;AACxD,OAAO,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AACpD,OAAO,EAAE,wBAAwB,EAAE,MAAM,mBAAmB,CAAC;AAC7D,OAAO,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAC;AAC5D,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvD,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAC9D,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACpD,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAiBnD,MAAM,CAAC,KAAK,UAAU,qBAAqB,CACzC,SAA0B,mBAAmB,EAAE;IAE/C,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;IAChE,MAAM,MAAM,GAAG,IAAI,cAAc,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;IACrD,MAAM,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC/B,MAAM,KAAK,GAAG,IAAI,cAAc,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;IAC3D,MAAM,YAAY,GAAG,IAAI,YAAY,CAAC,KAAK,CAAC,CAAC;IAC7C,MAAM,YAAY,GAA0B;QAC1C,MAAM;QACN,QAAQ;QACR,MAAM;QACN,KAAK;QACL,YAAY;KACb,CAAC;IAEF,MAAM,GAAG,GAAG,OAAO,CAAC;QAClB,MAAM,EAAE,IAAI;KACb,CAAC,CAAC;IAEH,mBAAmB,CAAC,GAAG,CAAC,CAAC;IACzB,mBAAmB,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;IACvC,mBAAmB,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;IACvC,iBAAiB,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;IACrC,oBAAoB,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;IACxC,wBAAwB,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;IAE5C,GAAG,CAAC,eAAe,CAAC,CAAC,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,EAAE;QAC7C,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE,0BAA0B,CAAC,CAAC;QAC1D,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACvE,KAAK,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;YACxB,KAAK,EAAE,OAAO;SACf,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,OAAO;QACL,GAAG;QACH,YAAY;QACZ,KAAK,CAAC,KAAK;YACT,YAAY,CAAC,KAAK,EAAE,CAAC;YACrB,MAAM,GAAG,CAAC,MAAM,CAAC;gBACf,IAAI,EAAE,MAAM,CAAC,IAAI;gBACjB,IAAI,EAAE,SAAS;aAChB,CAAC,CAAC;QACL,CAAC;QACD,KAAK,CAAC,KAAK;YACT,YAAY,CAAC,IAAI,EAAE,CAAC;YACpB,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;YACpB,MAAM,GAAG,CAAC,KAAK,EAAE,CAAC;YAClB,QAAQ,CAAC,KAAK,EAAE,CAAC;QACnB,CAAC;KACF,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { type QueryRow } from "@wide-events/internal";
|
|
2
|
+
export declare class DuckDbDatabase {
|
|
3
|
+
private readonly instance;
|
|
4
|
+
private readonly writer;
|
|
5
|
+
private constructor();
|
|
6
|
+
static create(path: string): Promise<DuckDbDatabase>;
|
|
7
|
+
execute(sql: string, values?: readonly unknown[]): Promise<void>;
|
|
8
|
+
executeRead(sql: string, values?: readonly unknown[]): Promise<QueryRow[]>;
|
|
9
|
+
executeWriteQuery(sql: string, values?: readonly unknown[]): Promise<QueryRow[]>;
|
|
10
|
+
close(): void;
|
|
11
|
+
}
|
|
12
|
+
//# sourceMappingURL=database.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"database.d.ts","sourceRoot":"","sources":["../../src/storage/database.ts"],"names":[],"mappings":"AAKA,OAAO,EAAkB,KAAK,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AAEtE,qBAAa,cAAc;IAEvB,OAAO,CAAC,QAAQ,CAAC,QAAQ;IACzB,OAAO,CAAC,QAAQ,CAAC,MAAM;IAFzB,OAAO;WAKM,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,CAAC;IAQpD,OAAO,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,GAAE,SAAS,OAAO,EAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAIpE,WAAW,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,GAAE,SAAS,OAAO,EAAO,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;IAa9E,iBAAiB,CACrB,GAAG,EAAE,MAAM,EACX,MAAM,GAAE,SAAS,OAAO,EAAO,GAC9B,OAAO,CAAC,QAAQ,EAAE,CAAC;IAKtB,KAAK,IAAI,IAAI;CAId"}
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
import { DuckDBInstance } from "@duckdb/node-api";
|
|
2
|
+
import { BASE_TABLE_SQL } from "@wide-events/internal";
|
|
3
|
+
export class DuckDbDatabase {
|
|
4
|
+
instance;
|
|
5
|
+
writer;
|
|
6
|
+
constructor(instance, writer) {
|
|
7
|
+
this.instance = instance;
|
|
8
|
+
this.writer = writer;
|
|
9
|
+
}
|
|
10
|
+
static async create(path) {
|
|
11
|
+
const instance = await DuckDBInstance.create(path);
|
|
12
|
+
const writer = await instance.connect();
|
|
13
|
+
const database = new DuckDbDatabase(instance, writer);
|
|
14
|
+
await database.writer.run(BASE_TABLE_SQL);
|
|
15
|
+
return database;
|
|
16
|
+
}
|
|
17
|
+
async execute(sql, values = []) {
|
|
18
|
+
await this.writer.run(sql, toDuckDbValues(values));
|
|
19
|
+
}
|
|
20
|
+
async executeRead(sql, values = []) {
|
|
21
|
+
const readerConnection = await this.instance.connect();
|
|
22
|
+
try {
|
|
23
|
+
const reader = await readerConnection.runAndReadAll(sql, toDuckDbValues(values));
|
|
24
|
+
return normalizeRows(reader.getRowObjectsJS());
|
|
25
|
+
}
|
|
26
|
+
finally {
|
|
27
|
+
readerConnection.closeSync();
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
async executeWriteQuery(sql, values = []) {
|
|
31
|
+
const reader = await this.writer.runAndReadAll(sql, toDuckDbValues(values));
|
|
32
|
+
return normalizeRows(reader.getRowObjectsJS());
|
|
33
|
+
}
|
|
34
|
+
close() {
|
|
35
|
+
this.writer.closeSync();
|
|
36
|
+
this.instance.closeSync();
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
function normalizeRows(rows) {
|
|
40
|
+
return rows.map((row) => {
|
|
41
|
+
const normalized = {};
|
|
42
|
+
for (const [key, value] of Object.entries(row)) {
|
|
43
|
+
normalized[key] = normalizeResultValue(value);
|
|
44
|
+
}
|
|
45
|
+
return normalized;
|
|
46
|
+
});
|
|
47
|
+
}
|
|
48
|
+
function normalizeResultValue(value) {
|
|
49
|
+
if (value === null ||
|
|
50
|
+
typeof value === "string" ||
|
|
51
|
+
typeof value === "number" ||
|
|
52
|
+
typeof value === "boolean") {
|
|
53
|
+
return value;
|
|
54
|
+
}
|
|
55
|
+
if (typeof value === "bigint") {
|
|
56
|
+
const numericValue = Number(value);
|
|
57
|
+
return Number.isSafeInteger(numericValue) ? numericValue : value.toString();
|
|
58
|
+
}
|
|
59
|
+
if (value instanceof Date) {
|
|
60
|
+
return value.toISOString();
|
|
61
|
+
}
|
|
62
|
+
return JSON.stringify(value);
|
|
63
|
+
}
|
|
64
|
+
function toDuckDbValues(values) {
|
|
65
|
+
return values.map((value) => {
|
|
66
|
+
if (value === null ||
|
|
67
|
+
typeof value === "string" ||
|
|
68
|
+
typeof value === "number" ||
|
|
69
|
+
typeof value === "boolean" ||
|
|
70
|
+
typeof value === "bigint") {
|
|
71
|
+
return value;
|
|
72
|
+
}
|
|
73
|
+
if (value instanceof Date) {
|
|
74
|
+
return value.toISOString();
|
|
75
|
+
}
|
|
76
|
+
if (typeof value === "undefined") {
|
|
77
|
+
return null;
|
|
78
|
+
}
|
|
79
|
+
throw new Error(`Unsupported DuckDB parameter type: ${typeof value}`);
|
|
80
|
+
});
|
|
81
|
+
}
|
|
82
|
+
//# sourceMappingURL=database.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"database.js","sourceRoot":"","sources":["../../src/storage/database.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,cAAc,EAEf,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,cAAc,EAAiB,MAAM,uBAAuB,CAAC;AAEtE,MAAM,OAAO,cAAc;IAEN;IACA;IAFnB,YACmB,QAAwB,EACxB,MAAwB;QADxB,aAAQ,GAAR,QAAQ,CAAgB;QACxB,WAAM,GAAN,MAAM,CAAkB;IACxC,CAAC;IAEJ,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,IAAY;QAC9B,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QACnD,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,OAAO,EAAE,CAAC;QACxC,MAAM,QAAQ,GAAG,IAAI,cAAc,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QACtD,MAAM,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;QAC1C,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,GAAW,EAAE,SAA6B,EAAE;QACxD,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC;IACrD,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,GAAW,EAAE,SAA6B,EAAE;QAC5D,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC;QACvD,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,gBAAgB,CAAC,aAAa,CACjD,GAAG,EACH,cAAc,CAAC,MAAM,CAAC,CACvB,CAAC;YACF,OAAO,aAAa,CAAC,MAAM,CAAC,eAAe,EAAE,CAAC,CAAC;QACjD,CAAC;gBAAS,CAAC;YACT,gBAAgB,CAAC,SAAS,EAAE,CAAC;QAC/B,CAAC;IACH,CAAC;IAED,KAAK,CAAC,iBAAiB,CACrB,GAAW,EACX,SAA6B,EAAE;QAE/B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,GAAG,EAAE,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC;QAC5E,OAAO,aAAa,CAAC,MAAM,CAAC,eAAe,EAAE,CAAC,CAAC;IACjD,CAAC;IAED,KAAK;QACH,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;QACxB,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC;IAC5B,CAAC;CACF;AAED,SAAS,aAAa,CAAC,IAA+B;IACpD,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;QACtB,MAAM,UAAU,GAAa,EAAE,CAAC;QAChC,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;YAC/C,UAAU,CAAC,GAAG,CAAC,GAAG,oBAAoB,CAAC,KAAK,CAAC,CAAC;QAChD,CAAC;QACD,OAAO,UAAU,CAAC;IACpB,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,oBAAoB,CAAC,KAAc;IAC1C,IACE,KAAK,KAAK,IAAI;QACd,OAAO,KAAK,KAAK,QAAQ;QACzB,OAAO,KAAK,KAAK,QAAQ;QACzB,OAAO,KAAK,KAAK,SAAS,EAC1B,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAED,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC9B,MAAM,YAAY,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;QACnC,OAAO,MAAM,CAAC,aAAa,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC;IAC9E,CAAC;IAED,IAAI,KAAK,YAAY,IAAI,EAAE,CAAC;QAC1B,OAAO,KAAK,CAAC,WAAW,EAAE,CAAC;IAC7B,CAAC;IAED,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;AAC/B,CAAC;AAED,SAAS,cAAc,CAAC,MAA0B;IAChD,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;QAC1B,IACE,KAAK,KAAK,IAAI;YACd,OAAO,KAAK,KAAK,QAAQ;YACzB,OAAO,KAAK,KAAK,QAAQ;YACzB,OAAO,KAAK,KAAK,SAAS;YAC1B,OAAO,KAAK,KAAK,QAAQ,EACzB,CAAC;YACD,OAAO,KAAK,CAAC;QACf,CAAC;QAED,IAAI,KAAK,YAAY,IAAI,EAAE,CAAC;YAC1B,OAAO,KAAK,CAAC,WAAW,EAAE,CAAC;QAC7B,CAAC;QAED,IAAI,OAAO,KAAK,KAAK,WAAW,EAAE,CAAC;YACjC,OAAO,IAAI,CAAC;QACd,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,sCAAsC,OAAO,KAAK,EAAE,CAAC,CAAC;IACxE,CAAC,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { type ColumnInfo } from "@wide-events/internal";
|
|
2
|
+
import { DuckDbDatabase } from "./database.js";
|
|
3
|
+
export declare class SchemaRegistry {
|
|
4
|
+
private readonly maxColumns;
|
|
5
|
+
private readonly columns;
|
|
6
|
+
constructor(maxColumns: number);
|
|
7
|
+
hydrate(database: DuckDbDatabase): Promise<void>;
|
|
8
|
+
listColumns(): ColumnInfo[];
|
|
9
|
+
isKnownColumn(name: string): boolean;
|
|
10
|
+
ensureDynamicColumns(database: DuckDbDatabase, candidateColumns: readonly string[]): Promise<string[]>;
|
|
11
|
+
}
|
|
12
|
+
//# sourceMappingURL=schema-registry.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"schema-registry.d.ts","sourceRoot":"","sources":["../../src/storage/schema-registry.ts"],"names":[],"mappings":"AAAA,OAAO,EAGL,KAAK,UAAU,EAGhB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAE/C,qBAAa,cAAc;IAGb,OAAO,CAAC,QAAQ,CAAC,UAAU;IAFvC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAiC;gBAE5B,UAAU,EAAE,MAAM;IAUzC,OAAO,CAAC,QAAQ,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC;IActD,WAAW,IAAI,UAAU,EAAE;IAM3B,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO;IAI9B,oBAAoB,CACxB,QAAQ,EAAE,cAAc,EACxB,gBAAgB,EAAE,SAAS,MAAM,EAAE,GAClC,OAAO,CAAC,MAAM,EAAE,CAAC;CAyBrB"}
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import { BASELINE_COLUMN_NAMES, BASELINE_COLUMN_TYPES, quoteIdentifier } from "@wide-events/internal";
|
|
2
|
+
export class SchemaRegistry {
|
|
3
|
+
maxColumns;
|
|
4
|
+
columns = new Map();
|
|
5
|
+
constructor(maxColumns) {
|
|
6
|
+
this.maxColumns = maxColumns;
|
|
7
|
+
for (const [name, type] of Object.entries(BASELINE_COLUMN_TYPES)) {
|
|
8
|
+
this.columns.set(name, {
|
|
9
|
+
name,
|
|
10
|
+
type,
|
|
11
|
+
origin: "baseline"
|
|
12
|
+
});
|
|
13
|
+
}
|
|
14
|
+
}
|
|
15
|
+
async hydrate(database) {
|
|
16
|
+
const rows = await database.executeWriteQuery("PRAGMA table_info('events')");
|
|
17
|
+
for (const row of rows) {
|
|
18
|
+
const name = expectString(row["name"], "PRAGMA table_info.name");
|
|
19
|
+
const type = expectString(row["type"], "PRAGMA table_info.type");
|
|
20
|
+
this.columns.set(name, {
|
|
21
|
+
name,
|
|
22
|
+
type,
|
|
23
|
+
origin: BASELINE_COLUMN_NAMES.includes(name) ? "baseline" : "dynamic"
|
|
24
|
+
});
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
listColumns() {
|
|
28
|
+
return [...this.columns.values()].sort((left, right) => left.name.localeCompare(right.name));
|
|
29
|
+
}
|
|
30
|
+
isKnownColumn(name) {
|
|
31
|
+
return this.columns.has(name);
|
|
32
|
+
}
|
|
33
|
+
async ensureDynamicColumns(database, candidateColumns) {
|
|
34
|
+
const dropped = [];
|
|
35
|
+
for (const column of candidateColumns) {
|
|
36
|
+
if (this.columns.has(column)) {
|
|
37
|
+
continue;
|
|
38
|
+
}
|
|
39
|
+
if (this.columns.size >= this.maxColumns) {
|
|
40
|
+
dropped.push(column);
|
|
41
|
+
continue;
|
|
42
|
+
}
|
|
43
|
+
await database.execute(`ALTER TABLE events ADD COLUMN IF NOT EXISTS ${quoteIdentifier(column)} VARCHAR`);
|
|
44
|
+
this.columns.set(column, {
|
|
45
|
+
name: column,
|
|
46
|
+
type: "VARCHAR",
|
|
47
|
+
origin: "dynamic"
|
|
48
|
+
});
|
|
49
|
+
}
|
|
50
|
+
return dropped;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
function expectString(value, label) {
|
|
54
|
+
if (typeof value !== "string") {
|
|
55
|
+
throw new Error(`${label} must be a string`);
|
|
56
|
+
}
|
|
57
|
+
return value;
|
|
58
|
+
}
|
|
59
|
+
//# sourceMappingURL=schema-registry.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"schema-registry.js","sourceRoot":"","sources":["../../src/storage/schema-registry.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,qBAAqB,EACrB,qBAAqB,EAErB,eAAe,EAEhB,MAAM,uBAAuB,CAAC;AAG/B,MAAM,OAAO,cAAc;IAGI;IAFZ,OAAO,GAAG,IAAI,GAAG,EAAsB,CAAC;IAEzD,YAA6B,UAAkB;QAAlB,eAAU,GAAV,UAAU,CAAQ;QAC7C,KAAK,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,qBAAqB,CAAC,EAAE,CAAC;YACjE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE;gBACrB,IAAI;gBACJ,IAAI;gBACJ,MAAM,EAAE,UAAU;aACnB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,QAAwB;QACpC,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,iBAAiB,CAAC,6BAA6B,CAAC,CAAC;QAE7E,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,MAAM,IAAI,GAAG,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,wBAAwB,CAAC,CAAC;YACjE,MAAM,IAAI,GAAG,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,wBAAwB,CAAC,CAAC;YACjE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE;gBACrB,IAAI;gBACJ,IAAI;gBACJ,MAAM,EAAE,qBAAqB,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS;aACtE,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,WAAW;QACT,OAAO,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CACrD,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,CACpC,CAAC;IACJ,CAAC;IAED,aAAa,CAAC,IAAY;QACxB,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAChC,CAAC;IAED,KAAK,CAAC,oBAAoB,CACxB,QAAwB,EACxB,gBAAmC;QAEnC,MAAM,OAAO,GAAa,EAAE,CAAC;QAE7B,KAAK,MAAM,MAAM,IAAI,gBAAgB,EAAE,CAAC;YACtC,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC7B,SAAS;YACX,CAAC;YAED,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;gBACzC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBACrB,SAAS;YACX,CAAC;YAED,MAAM,QAAQ,CAAC,OAAO,CACpB,+CAA+C,eAAe,CAAC,MAAM,CAAC,UAAU,CACjF,CAAC;YACF,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE;gBACvB,IAAI,EAAE,MAAM;gBACZ,IAAI,EAAE,SAAS;gBACf,MAAM,EAAE,SAAS;aAClB,CAAC,CAAC;QACL,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;CACF;AAED,SAAS,YAAY,CAAC,KAAc,EAAE,KAAa;IACjD,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC9B,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,mBAAmB,CAAC,CAAC;IAC/C,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"serialized-executor.d.ts","sourceRoot":"","sources":["../../src/storage/serialized-executor.ts"],"names":[],"mappings":"AAAA,qBAAa,kBAAkB;IAC7B,OAAO,CAAC,IAAI,CAAoC;IAEhD,OAAO,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;CAQ/C"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"serialized-executor.js","sourceRoot":"","sources":["../../src/storage/serialized-executor.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,kBAAkB;IACrB,IAAI,GAAkB,OAAO,CAAC,OAAO,EAAE,CAAC;IAEhD,OAAO,CAAI,IAAsB;QAC/B,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QAC1C,IAAI,CAAC,IAAI,GAAG,MAAM,CAAC,IAAI,CACrB,GAAG,EAAE,CAAC,SAAS,EACf,GAAG,EAAE,CAAC,SAAS,CAChB,CAAC;QACF,OAAO,MAAM,CAAC;IAChB,CAAC;CACF"}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { type FlatEventRow } from "@wide-events/internal";
|
|
2
|
+
import type { CollectorConfig } from "../config.js";
|
|
3
|
+
import { DuckDbDatabase } from "./database.js";
|
|
4
|
+
import { SchemaRegistry } from "./schema-registry.js";
|
|
5
|
+
export declare class CollectorStore {
|
|
6
|
+
private readonly database;
|
|
7
|
+
private readonly schema;
|
|
8
|
+
private readonly config;
|
|
9
|
+
private readonly executor;
|
|
10
|
+
private readonly pending;
|
|
11
|
+
private flushTimer;
|
|
12
|
+
private pendingRowCount;
|
|
13
|
+
constructor(database: DuckDbDatabase, schema: SchemaRegistry, config: CollectorConfig);
|
|
14
|
+
enqueueRows(rows: readonly FlatEventRow[]): Promise<void>;
|
|
15
|
+
flush(): Promise<void>;
|
|
16
|
+
runRetention(now?: Date): Promise<void>;
|
|
17
|
+
private flushSoon;
|
|
18
|
+
}
|
|
19
|
+
//# sourceMappingURL=store.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"store.d.ts","sourceRoot":"","sources":["../../src/storage/store.ts"],"names":[],"mappings":"AAAA,OAAO,EAKL,KAAK,YAAY,EAClB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AACpD,OAAO,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAC/C,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAStD,qBAAa,cAAc;IAOvB,OAAO,CAAC,QAAQ,CAAC,QAAQ;IACzB,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,MAAM;IARzB,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAA4B;IACrD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAsB;IAC9C,OAAO,CAAC,UAAU,CAA6B;IAC/C,OAAO,CAAC,eAAe,CAAK;gBAGT,QAAQ,EAAE,cAAc,EACxB,MAAM,EAAE,cAAc,EACtB,MAAM,EAAE,eAAe;IAGpC,WAAW,CAAC,IAAI,EAAE,SAAS,YAAY,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IA8BzD,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAQtB,YAAY,CAAC,GAAG,GAAE,IAAiB,GAAG,OAAO,CAAC,IAAI,CAAC;YAW3C,SAAS;CAgCxB"}
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
import { BASELINE_COLUMN_NAMES, isBaselineColumn, quoteIdentifier, sanitizeIdentifier } from "@wide-events/internal";
|
|
2
|
+
import { SerializedExecutor } from "./serialized-executor.js";
|
|
3
|
+
export class CollectorStore {
|
|
4
|
+
database;
|
|
5
|
+
schema;
|
|
6
|
+
config;
|
|
7
|
+
executor = new SerializedExecutor();
|
|
8
|
+
pending = [];
|
|
9
|
+
flushTimer;
|
|
10
|
+
pendingRowCount = 0;
|
|
11
|
+
constructor(database, schema, config) {
|
|
12
|
+
this.database = database;
|
|
13
|
+
this.schema = schema;
|
|
14
|
+
this.config = config;
|
|
15
|
+
}
|
|
16
|
+
async enqueueRows(rows) {
|
|
17
|
+
if (rows.length === 0) {
|
|
18
|
+
return;
|
|
19
|
+
}
|
|
20
|
+
if (this.pendingRowCount + rows.length > this.config.queueLimit) {
|
|
21
|
+
throw new Error("Collector queue limit exceeded");
|
|
22
|
+
}
|
|
23
|
+
return await new Promise((resolve, reject) => {
|
|
24
|
+
this.pending.push({
|
|
25
|
+
rows: [...rows],
|
|
26
|
+
resolve,
|
|
27
|
+
reject
|
|
28
|
+
});
|
|
29
|
+
this.pendingRowCount += rows.length;
|
|
30
|
+
if (this.pendingRowCount >= this.config.batchSize) {
|
|
31
|
+
void this.flushSoon();
|
|
32
|
+
return;
|
|
33
|
+
}
|
|
34
|
+
if (!this.flushTimer) {
|
|
35
|
+
this.flushTimer = setTimeout(() => {
|
|
36
|
+
void this.flushSoon();
|
|
37
|
+
}, this.config.batchTimeoutMs);
|
|
38
|
+
}
|
|
39
|
+
});
|
|
40
|
+
}
|
|
41
|
+
async flush() {
|
|
42
|
+
if (this.pendingRowCount === 0) {
|
|
43
|
+
return;
|
|
44
|
+
}
|
|
45
|
+
await this.flushSoon();
|
|
46
|
+
}
|
|
47
|
+
async runRetention(now = new Date()) {
|
|
48
|
+
const cutoff = new Date(now.getTime() - this.config.retentionDays * 24 * 60 * 60 * 1_000).toISOString();
|
|
49
|
+
await this.executor.enqueue(async () => {
|
|
50
|
+
await this.database.execute("DELETE FROM events WHERE ts < ?", [cutoff]);
|
|
51
|
+
await this.database.execute("CHECKPOINT");
|
|
52
|
+
});
|
|
53
|
+
}
|
|
54
|
+
async flushSoon() {
|
|
55
|
+
if (this.flushTimer) {
|
|
56
|
+
clearTimeout(this.flushTimer);
|
|
57
|
+
this.flushTimer = undefined;
|
|
58
|
+
}
|
|
59
|
+
const batch = this.pending.splice(0, this.pending.length);
|
|
60
|
+
if (batch.length === 0) {
|
|
61
|
+
return;
|
|
62
|
+
}
|
|
63
|
+
const rows = batch.flatMap((entry) => entry.rows);
|
|
64
|
+
this.pendingRowCount -= rows.length;
|
|
65
|
+
try {
|
|
66
|
+
await this.executor.enqueue(async () => {
|
|
67
|
+
await this.schema.ensureDynamicColumns(this.database, collectDynamicColumns(rows));
|
|
68
|
+
await insertRows(this.database, rows);
|
|
69
|
+
});
|
|
70
|
+
for (const entry of batch) {
|
|
71
|
+
entry.resolve();
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
catch (error) {
|
|
75
|
+
for (const entry of batch) {
|
|
76
|
+
entry.reject(error);
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
function collectDynamicColumns(rows) {
|
|
82
|
+
const columnSet = new Set();
|
|
83
|
+
for (const row of rows) {
|
|
84
|
+
for (const key of Object.keys(row)) {
|
|
85
|
+
if (isBaselineColumn(key)) {
|
|
86
|
+
continue;
|
|
87
|
+
}
|
|
88
|
+
sanitizeIdentifier(key);
|
|
89
|
+
columnSet.add(key);
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
return [...columnSet].sort();
|
|
93
|
+
}
|
|
94
|
+
async function insertRows(database, rows) {
|
|
95
|
+
if (rows.length === 0) {
|
|
96
|
+
return;
|
|
97
|
+
}
|
|
98
|
+
const columnNames = collectInsertColumns(rows);
|
|
99
|
+
const placeholders = rows
|
|
100
|
+
.map(() => `(${columnNames.map(() => "?").join(", ")})`)
|
|
101
|
+
.join(", ");
|
|
102
|
+
const sql = `INSERT INTO events (${columnNames
|
|
103
|
+
.map((column) => quoteIdentifier(column))
|
|
104
|
+
.join(", ")}) VALUES ${placeholders}`;
|
|
105
|
+
const values = [];
|
|
106
|
+
for (const row of rows) {
|
|
107
|
+
for (const column of columnNames) {
|
|
108
|
+
values.push(serializeRowValue(row[column]));
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
await database.execute(sql, values);
|
|
112
|
+
}
|
|
113
|
+
function collectInsertColumns(rows) {
|
|
114
|
+
const columnSet = new Set(BASELINE_COLUMN_NAMES);
|
|
115
|
+
for (const row of rows) {
|
|
116
|
+
for (const key of Object.keys(row)) {
|
|
117
|
+
columnSet.add(key);
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
return [...columnSet].sort();
|
|
121
|
+
}
|
|
122
|
+
function serializeRowValue(value) {
|
|
123
|
+
if (value === null ||
|
|
124
|
+
typeof value === "string" ||
|
|
125
|
+
typeof value === "number" ||
|
|
126
|
+
typeof value === "boolean") {
|
|
127
|
+
return value;
|
|
128
|
+
}
|
|
129
|
+
if (typeof value === "undefined") {
|
|
130
|
+
return null;
|
|
131
|
+
}
|
|
132
|
+
return JSON.stringify(value);
|
|
133
|
+
}
|
|
134
|
+
//# sourceMappingURL=store.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"store.js","sourceRoot":"","sources":["../../src/storage/store.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,qBAAqB,EACrB,gBAAgB,EAChB,eAAe,EACf,kBAAkB,EAEnB,MAAM,uBAAuB,CAAC;AAI/B,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AAQ9D,MAAM,OAAO,cAAc;IAON;IACA;IACA;IARF,QAAQ,GAAG,IAAI,kBAAkB,EAAE,CAAC;IACpC,OAAO,GAAmB,EAAE,CAAC;IACtC,UAAU,CAA6B;IACvC,eAAe,GAAG,CAAC,CAAC;IAE5B,YACmB,QAAwB,EACxB,MAAsB,EACtB,MAAuB;QAFvB,aAAQ,GAAR,QAAQ,CAAgB;QACxB,WAAM,GAAN,MAAM,CAAgB;QACtB,WAAM,GAAN,MAAM,CAAiB;IACvC,CAAC;IAEJ,KAAK,CAAC,WAAW,CAAC,IAA6B;QAC7C,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtB,OAAO;QACT,CAAC;QAED,IAAI,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC;YAChE,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;QACpD,CAAC;QAED,OAAO,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACjD,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;gBAChB,IAAI,EAAE,CAAC,GAAG,IAAI,CAAC;gBACf,OAAO;gBACP,MAAM;aACP,CAAC,CAAC;YACH,IAAI,CAAC,eAAe,IAAI,IAAI,CAAC,MAAM,CAAC;YAEpC,IAAI,IAAI,CAAC,eAAe,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;gBAClD,KAAK,IAAI,CAAC,SAAS,EAAE,CAAC;gBACtB,OAAO;YACT,CAAC;YAED,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;gBACrB,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC,GAAG,EAAE;oBAChC,KAAK,IAAI,CAAC,SAAS,EAAE,CAAC;gBACxB,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;YACjC,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,eAAe,KAAK,CAAC,EAAE,CAAC;YAC/B,OAAO;QACT,CAAC;QAED,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;IACzB,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,MAAY,IAAI,IAAI,EAAE;QACvC,MAAM,MAAM,GAAG,IAAI,IAAI,CACrB,GAAG,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,aAAa,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,KAAK,CACjE,CAAC,WAAW,EAAE,CAAC;QAEhB,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,IAAI,EAAE;YACrC,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,iCAAiC,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC;YACzE,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;QAC5C,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,SAAS;QACrB,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,YAAY,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAC9B,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;QAC9B,CAAC;QAED,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAC1D,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO;QACT,CAAC;QAED,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAClD,IAAI,CAAC,eAAe,IAAI,IAAI,CAAC,MAAM,CAAC;QAEpC,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,IAAI,EAAE;gBACrC,MAAM,IAAI,CAAC,MAAM,CAAC,oBAAoB,CACpC,IAAI,CAAC,QAAQ,EACb,qBAAqB,CAAC,IAAI,CAAC,CAC5B,CAAC;gBACF,MAAM,UAAU,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;YACxC,CAAC,CAAC,CAAC;YAEH,KAAK,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;gBAC1B,KAAK,CAAC,OAAO,EAAE,CAAC;YAClB,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,KAAK,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;gBAC1B,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACtB,CAAC;QACH,CAAC;IACH,CAAC;CACF;AAED,SAAS,qBAAqB,CAAC,IAA6B;IAC1D,MAAM,SAAS,GAAG,IAAI,GAAG,EAAU,CAAC;IACpC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;YACnC,IAAI,gBAAgB,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC1B,SAAS;YACX,CAAC;YAED,kBAAkB,CAAC,GAAG,CAAC,CAAC;YACxB,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACrB,CAAC;IACH,CAAC;IAED,OAAO,CAAC,GAAG,SAAS,CAAC,CAAC,IAAI,EAAE,CAAC;AAC/B,CAAC;AAED,KAAK,UAAU,UAAU,CACvB,QAAwB,EACxB,IAA6B;IAE7B,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACtB,OAAO;IACT,CAAC;IAED,MAAM,WAAW,GAAG,oBAAoB,CAAC,IAAI,CAAC,CAAC;IAC/C,MAAM,YAAY,GAAG,IAAI;SACtB,GAAG,CAAC,GAAG,EAAE,CAAC,IAAI,WAAW,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC;SACvD,IAAI,CAAC,IAAI,CAAC,CAAC;IACd,MAAM,GAAG,GAAG,uBAAuB,WAAW;SAC3C,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;SACxC,IAAI,CAAC,IAAI,CAAC,YAAY,YAAY,EAAE,CAAC;IAExC,MAAM,MAAM,GAAc,EAAE,CAAC;IAC7B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,KAAK,MAAM,MAAM,IAAI,WAAW,EAAE,CAAC;YACjC,MAAM,CAAC,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAC9C,CAAC;IACH,CAAC;IAED,MAAM,QAAQ,CAAC,OAAO,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;AACtC,CAAC;AAED,SAAS,oBAAoB,CAAC,IAA6B;IACzD,MAAM,SAAS,GAAG,IAAI,GAAG,CAAS,qBAAqB,CAAC,CAAC;IACzD,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;YACnC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACrB,CAAC;IACH,CAAC;IAED,OAAO,CAAC,GAAG,SAAS,CAAC,CAAC,IAAI,EAAE,CAAC;AAC/B,CAAC;AAED,SAAS,iBAAiB,CAAC,KAAc;IACvC,IACE,KAAK,KAAK,IAAI;QACd,OAAO,KAAK,KAAK,QAAQ;QACzB,OAAO,KAAK,KAAK,QAAQ;QACzB,OAAO,KAAK,KAAK,SAAS,EAC1B,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAED,IAAI,OAAO,KAAK,KAAK,WAAW,EAAE,CAAC;QACjC,OAAO,IAAI,CAAC;IACd,CAAC;IAED,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;AAC/B,CAAC"}
|
package/package.json
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@wide-events/collector",
|
|
3
|
+
"version": "0.1.0",
|
|
4
|
+
"type": "module",
|
|
5
|
+
"license": "AGPL-3.0-only",
|
|
6
|
+
"main": "./dist/index.js",
|
|
7
|
+
"types": "./dist/index.d.ts",
|
|
8
|
+
"bin": {
|
|
9
|
+
"wide-events-collector": "./dist/cli.js"
|
|
10
|
+
},
|
|
11
|
+
"exports": {
|
|
12
|
+
".": {
|
|
13
|
+
"types": "./dist/index.d.ts",
|
|
14
|
+
"import": "./dist/index.js",
|
|
15
|
+
"default": "./dist/index.js"
|
|
16
|
+
}
|
|
17
|
+
},
|
|
18
|
+
"publishConfig": {
|
|
19
|
+
"access": "public"
|
|
20
|
+
},
|
|
21
|
+
"files": [
|
|
22
|
+
"dist",
|
|
23
|
+
"README.md",
|
|
24
|
+
"LICENSE"
|
|
25
|
+
],
|
|
26
|
+
"dependencies": {
|
|
27
|
+
"@duckdb/node-api": "1.5.0-r.1",
|
|
28
|
+
"fastify": "5.8.2",
|
|
29
|
+
"zod": "4.3.6",
|
|
30
|
+
"@wide-events/internal": "0.1.0"
|
|
31
|
+
},
|
|
32
|
+
"scripts": {
|
|
33
|
+
"build": "tsc -p tsconfig.json",
|
|
34
|
+
"typecheck": "tsc -p tsconfig.json --noEmit",
|
|
35
|
+
"clean": "rm -rf dist tsconfig.tsbuildinfo"
|
|
36
|
+
}
|
|
37
|
+
}
|