lduck 0.0.2 → 0.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -3
- package/dist/index.js +405 -114
- package/dist/static/client.js +9 -9
- package/package.json +2 -3
package/dist/index.js
CHANGED
|
@@ -53,9 +53,64 @@ function createApp(scriptSrc, setup) {
|
|
|
53
53
|
});
|
|
54
54
|
return app;
|
|
55
55
|
}
|
|
56
|
+
const LOG_LEVELS = [
|
|
57
|
+
"DEBUG",
|
|
58
|
+
"INFO",
|
|
59
|
+
"WARN",
|
|
60
|
+
"ERROR",
|
|
61
|
+
"FATAL"
|
|
62
|
+
];
|
|
63
|
+
const FIELD_MAPPINGS = {
|
|
64
|
+
timestamp: ["timestamp", "ts", "time", "@timestamp", "datetime"],
|
|
65
|
+
level: ["level", "severity", "loglevel", "lvl", "priority"],
|
|
66
|
+
message: ["message", "msg", "body", "text"],
|
|
67
|
+
service: ["service", "svc", "app", "component", "logger"],
|
|
68
|
+
trace_id: ["trace_id", "traceId", "request_id", "correlation_id"]
|
|
69
|
+
};
|
|
70
|
+
const REVERSE_FIELD_MAP = (() => {
|
|
71
|
+
const map = /* @__PURE__ */ new Map();
|
|
72
|
+
for (const [canonical, aliases] of Object.entries(FIELD_MAPPINGS)) {
|
|
73
|
+
for (const alias of aliases) {
|
|
74
|
+
map.set(alias, canonical);
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
return map;
|
|
78
|
+
})();
|
|
79
|
+
function resolveField(fieldName) {
|
|
80
|
+
return REVERSE_FIELD_MAP.get(fieldName) ?? null;
|
|
81
|
+
}
|
|
82
|
+
function splitLevels(csv) {
|
|
83
|
+
if (!csv) return void 0;
|
|
84
|
+
const levels = csv.split(",").filter((v) => LOG_LEVELS.includes(v));
|
|
85
|
+
return levels.length > 0 ? levels : void 0;
|
|
86
|
+
}
|
|
87
|
+
function splitStrings(csv) {
|
|
88
|
+
if (!csv) return void 0;
|
|
89
|
+
const values = csv.split(",").filter(Boolean);
|
|
90
|
+
return values.length > 0 ? values : void 0;
|
|
91
|
+
}
|
|
92
|
+
function parseJsonFilters(raw) {
|
|
93
|
+
if (!raw) return void 0;
|
|
94
|
+
try {
|
|
95
|
+
const parsed = JSON.parse(raw);
|
|
96
|
+
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) return void 0;
|
|
97
|
+
const result = {};
|
|
98
|
+
for (const [key, value] of Object.entries(parsed)) {
|
|
99
|
+
if (Array.isArray(value)) {
|
|
100
|
+
result[key] = value.map(String);
|
|
101
|
+
} else if (typeof value === "string") {
|
|
102
|
+
result[key] = [value];
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
return Object.keys(result).length > 0 ? result : void 0;
|
|
106
|
+
} catch {
|
|
107
|
+
return void 0;
|
|
108
|
+
}
|
|
109
|
+
}
|
|
56
110
|
const logQuerySchema = z.object({
|
|
57
|
-
level: z.
|
|
111
|
+
level: z.string().optional(),
|
|
58
112
|
service: z.string().optional(),
|
|
113
|
+
host: z.string().optional(),
|
|
59
114
|
source: z.string().optional(),
|
|
60
115
|
search: z.string().optional(),
|
|
61
116
|
startTime: z.coerce.date().optional(),
|
|
@@ -73,7 +128,7 @@ const sqlQuerySchema = z.object({
|
|
|
73
128
|
});
|
|
74
129
|
const exportSchema = z.object({
|
|
75
130
|
format: z.enum(["csv", "json"]),
|
|
76
|
-
level: z.
|
|
131
|
+
level: z.string().optional(),
|
|
77
132
|
service: z.string().optional(),
|
|
78
133
|
source: z.string().optional(),
|
|
79
134
|
search: z.string().optional(),
|
|
@@ -87,14 +142,26 @@ const ingestBodySchema = z.union([
|
|
|
87
142
|
const facetQuerySchema = z.object({
|
|
88
143
|
field: z.string(),
|
|
89
144
|
jsonPath: z.string().optional(),
|
|
90
|
-
level: z.
|
|
145
|
+
level: z.string().optional(),
|
|
91
146
|
service: z.string().optional(),
|
|
147
|
+
host: z.string().optional(),
|
|
92
148
|
source: z.string().optional(),
|
|
93
149
|
search: z.string().optional(),
|
|
94
150
|
startTime: z.coerce.date().optional(),
|
|
95
151
|
endTime: z.coerce.date().optional(),
|
|
96
152
|
jsonFilters: z.string().optional()
|
|
97
153
|
});
|
|
154
|
+
const histogramQuerySchema = z.object({
|
|
155
|
+
buckets: z.coerce.number().int().min(1).max(360).default(30),
|
|
156
|
+
startTime: z.coerce.date().optional(),
|
|
157
|
+
endTime: z.coerce.date().optional(),
|
|
158
|
+
level: z.string().optional(),
|
|
159
|
+
service: z.string().optional(),
|
|
160
|
+
host: z.string().optional(),
|
|
161
|
+
source: z.string().optional(),
|
|
162
|
+
search: z.string().optional(),
|
|
163
|
+
jsonFilters: z.string().optional()
|
|
164
|
+
});
|
|
98
165
|
function validationHook(result, c) {
|
|
99
166
|
if (!result.success) {
|
|
100
167
|
return c.json(
|
|
@@ -132,7 +199,8 @@ function serializeStats$1(stats) {
|
|
|
132
199
|
timeRange: {
|
|
133
200
|
min: stats.timeRange.min?.toISOString() ?? null,
|
|
134
201
|
max: stats.timeRange.max?.toISOString() ?? null
|
|
135
|
-
}
|
|
202
|
+
},
|
|
203
|
+
ingestionRate: stats.ingestionRate
|
|
136
204
|
};
|
|
137
205
|
}
|
|
138
206
|
function createApiApp(db, ingester) {
|
|
@@ -159,6 +227,10 @@ function createApiApp(db, ingester) {
|
|
|
159
227
|
try {
|
|
160
228
|
const { source } = c.req.valid("query");
|
|
161
229
|
const stats = await db.getStats(source ? { source } : void 0);
|
|
230
|
+
if (ingester) {
|
|
231
|
+
const ingestionStats = ingester.getIngestionStats();
|
|
232
|
+
stats.ingestionRate = ingestionStats.ingestionRate;
|
|
233
|
+
}
|
|
162
234
|
return c.json(serializeStats$1(stats));
|
|
163
235
|
} catch (err) {
|
|
164
236
|
return c.json(
|
|
@@ -174,24 +246,18 @@ function createApiApp(db, ingester) {
|
|
|
174
246
|
}).get("/logs", zValidator("query", logQuerySchema, validationHook), async (c) => {
|
|
175
247
|
try {
|
|
176
248
|
const params = c.req.valid("query");
|
|
177
|
-
let jsonFilters;
|
|
178
|
-
if (params.jsonFilters) {
|
|
179
|
-
try {
|
|
180
|
-
jsonFilters = JSON.parse(params.jsonFilters);
|
|
181
|
-
} catch {
|
|
182
|
-
}
|
|
183
|
-
}
|
|
184
249
|
const queryParams = {
|
|
185
|
-
level: params.level,
|
|
186
|
-
service: params.service,
|
|
187
|
-
|
|
250
|
+
level: splitLevels(params.level),
|
|
251
|
+
service: splitStrings(params.service),
|
|
252
|
+
host: splitStrings(params.host),
|
|
253
|
+
source: splitStrings(params.source),
|
|
188
254
|
search: params.search,
|
|
189
255
|
startTime: params.startTime,
|
|
190
256
|
endTime: params.endTime,
|
|
191
257
|
limit: params.limit,
|
|
192
258
|
offset: params.offset,
|
|
193
259
|
order: params.order,
|
|
194
|
-
jsonFilters
|
|
260
|
+
jsonFilters: parseJsonFilters(params.jsonFilters)
|
|
195
261
|
};
|
|
196
262
|
const result = await db.queryLogs(queryParams);
|
|
197
263
|
return c.json({
|
|
@@ -209,6 +275,32 @@ function createApiApp(db, ingester) {
|
|
|
209
275
|
500
|
|
210
276
|
);
|
|
211
277
|
}
|
|
278
|
+
}).get("/histogram", zValidator("query", histogramQuerySchema, validationHook), async (c) => {
|
|
279
|
+
try {
|
|
280
|
+
const params = c.req.valid("query");
|
|
281
|
+
const result = await db.getHistogram({
|
|
282
|
+
buckets: params.buckets,
|
|
283
|
+
startTime: params.startTime,
|
|
284
|
+
endTime: params.endTime,
|
|
285
|
+
level: splitLevels(params.level),
|
|
286
|
+
service: splitStrings(params.service),
|
|
287
|
+
host: splitStrings(params.host),
|
|
288
|
+
source: splitStrings(params.source),
|
|
289
|
+
search: params.search,
|
|
290
|
+
jsonFilters: parseJsonFilters(params.jsonFilters)
|
|
291
|
+
});
|
|
292
|
+
return c.json(result);
|
|
293
|
+
} catch (err) {
|
|
294
|
+
return c.json(
|
|
295
|
+
{
|
|
296
|
+
error: {
|
|
297
|
+
code: "QUERY_ERROR",
|
|
298
|
+
message: err instanceof Error ? err.message : "Unknown error"
|
|
299
|
+
}
|
|
300
|
+
},
|
|
301
|
+
500
|
|
302
|
+
);
|
|
303
|
+
}
|
|
212
304
|
}).post("/query", zValidator("json", sqlQuerySchema, validationHook), async (c) => {
|
|
213
305
|
try {
|
|
214
306
|
const { sql } = c.req.valid("json");
|
|
@@ -241,9 +333,9 @@ function createApiApp(db, ingester) {
|
|
|
241
333
|
try {
|
|
242
334
|
const params = c.req.valid("json");
|
|
243
335
|
const queryParams = {
|
|
244
|
-
level: params.level,
|
|
245
|
-
service: params.service,
|
|
246
|
-
source: params.source,
|
|
336
|
+
level: splitLevels(params.level),
|
|
337
|
+
service: splitStrings(params.service),
|
|
338
|
+
source: splitStrings(params.source),
|
|
247
339
|
search: params.search,
|
|
248
340
|
startTime: params.startTime,
|
|
249
341
|
endTime: params.endTime,
|
|
@@ -293,21 +385,15 @@ function createApiApp(db, ingester) {
|
|
|
293
385
|
}).get("/facets", zValidator("query", facetQuerySchema, validationHook), async (c) => {
|
|
294
386
|
try {
|
|
295
387
|
const params = c.req.valid("query");
|
|
296
|
-
let jsonFilters;
|
|
297
|
-
if (params.jsonFilters) {
|
|
298
|
-
try {
|
|
299
|
-
jsonFilters = JSON.parse(params.jsonFilters);
|
|
300
|
-
} catch {
|
|
301
|
-
}
|
|
302
|
-
}
|
|
303
388
|
const filters = {
|
|
304
|
-
level: params.level,
|
|
305
|
-
service: params.service,
|
|
306
|
-
|
|
389
|
+
level: splitLevels(params.level),
|
|
390
|
+
service: splitStrings(params.service),
|
|
391
|
+
host: splitStrings(params.host),
|
|
392
|
+
source: splitStrings(params.source),
|
|
307
393
|
search: params.search,
|
|
308
394
|
startTime: params.startTime,
|
|
309
395
|
endTime: params.endTime,
|
|
310
|
-
jsonFilters
|
|
396
|
+
jsonFilters: parseJsonFilters(params.jsonFilters)
|
|
311
397
|
};
|
|
312
398
|
const distribution = await db.getFacetDistribution(
|
|
313
399
|
params.field,
|
|
@@ -333,6 +419,31 @@ new Hono().basePath("/api").get(
|
|
|
333
419
|
"/health",
|
|
334
420
|
(c) => c.json({ status: "ok", uptime: 0 })
|
|
335
421
|
);
|
|
422
|
+
const SEARCHABLE_FIELDS = /* @__PURE__ */ new Set([
|
|
423
|
+
"message",
|
|
424
|
+
"service",
|
|
425
|
+
"host",
|
|
426
|
+
"trace_id",
|
|
427
|
+
"level"
|
|
428
|
+
]);
|
|
429
|
+
function parseSearchQuery(search) {
|
|
430
|
+
const colonIndex = search.indexOf(":");
|
|
431
|
+
if (colonIndex <= 0) {
|
|
432
|
+
return { field: null, value: search };
|
|
433
|
+
}
|
|
434
|
+
const field = search.substring(0, colonIndex);
|
|
435
|
+
const value = search.substring(colonIndex + 1);
|
|
436
|
+
if (field.includes(" ")) {
|
|
437
|
+
return { field: null, value: search };
|
|
438
|
+
}
|
|
439
|
+
if (value === "") {
|
|
440
|
+
return { field: null, value: search };
|
|
441
|
+
}
|
|
442
|
+
if (!SEARCHABLE_FIELDS.has(field)) {
|
|
443
|
+
return { field: null, value: search };
|
|
444
|
+
}
|
|
445
|
+
return { field, value };
|
|
446
|
+
}
|
|
336
447
|
const CREATE_TABLE_SQL = `
|
|
337
448
|
CREATE TABLE IF NOT EXISTS logs (
|
|
338
449
|
_id BIGINT PRIMARY KEY,
|
|
@@ -457,12 +568,13 @@ class LogDatabase {
|
|
|
457
568
|
total: 0,
|
|
458
569
|
byLevel: {},
|
|
459
570
|
errorRate: 0,
|
|
460
|
-
timeRange: { min: null, max: null }
|
|
571
|
+
timeRange: { min: null, max: null },
|
|
572
|
+
ingestionRate: 0
|
|
461
573
|
};
|
|
462
574
|
}
|
|
463
575
|
const levelWhereClause = whereClause ? `${whereClause} AND level IS NOT NULL` : "WHERE level IS NOT NULL";
|
|
464
576
|
const levelReader = await conn.runAndReadAll(
|
|
465
|
-
`SELECT level, COUNT(*) as cnt FROM logs ${levelWhereClause} GROUP BY
|
|
577
|
+
`SELECT UPPER(level) as lvl, COUNT(*) as cnt FROM logs ${levelWhereClause} GROUP BY lvl ORDER BY cnt DESC`
|
|
466
578
|
);
|
|
467
579
|
const byLevel = {};
|
|
468
580
|
for (const row of levelReader.getRowsJS()) {
|
|
@@ -481,7 +593,8 @@ class LogDatabase {
|
|
|
481
593
|
timeRange: {
|
|
482
594
|
min: timeRow[0] !== null ? jsToDate(timeRow[0]) : null,
|
|
483
595
|
max: timeRow[1] !== null ? jsToDate(timeRow[1]) : null
|
|
484
|
-
}
|
|
596
|
+
},
|
|
597
|
+
ingestionRate: 0
|
|
485
598
|
};
|
|
486
599
|
}
|
|
487
600
|
// -------------------------------------------------------------------------
|
|
@@ -490,11 +603,13 @@ class LogDatabase {
|
|
|
490
603
|
async getFacetDistribution(field, jsonPath, filters) {
|
|
491
604
|
const conn = this.getConnection();
|
|
492
605
|
const columnExpr = jsonPath !== null ? `CAST(json_extract(_raw, '${escapeSql("$." + jsonPath)}') AS VARCHAR)` : `"${field}"`;
|
|
606
|
+
const isLevelField = field === "level" && jsonPath === null;
|
|
607
|
+
const selectExpr = isLevelField ? `UPPER(${columnExpr})` : columnExpr;
|
|
493
608
|
const conditions = buildFilterConditions(filters);
|
|
494
609
|
conditions.push(`${columnExpr} IS NOT NULL`);
|
|
495
610
|
const whereClause = `WHERE ${conditions.join(" AND ")}`;
|
|
496
611
|
const reader = await conn.runAndReadAll(
|
|
497
|
-
`SELECT ${
|
|
612
|
+
`SELECT ${selectExpr} as val, COUNT(*) as cnt FROM logs ${whereClause} GROUP BY val ORDER BY cnt DESC, val ASC`
|
|
498
613
|
);
|
|
499
614
|
const values = reader.getRowsJS().map((row) => {
|
|
500
615
|
let value = String(row[0]);
|
|
@@ -506,6 +621,59 @@ class LogDatabase {
|
|
|
506
621
|
return { field, values };
|
|
507
622
|
}
|
|
508
623
|
// -------------------------------------------------------------------------
|
|
624
|
+
// Histogram
|
|
625
|
+
// -------------------------------------------------------------------------
|
|
626
|
+
async getHistogram(params) {
|
|
627
|
+
const conn = this.getConnection();
|
|
628
|
+
const buckets = Math.min(360, Math.max(1, Math.floor(params.buckets ?? 30)));
|
|
629
|
+
const endTime = params.endTime ?? /* @__PURE__ */ new Date();
|
|
630
|
+
const startTime = params.startTime ?? new Date(endTime.getTime() - 60 * 60 * 1e3);
|
|
631
|
+
const rangeStartMs = Math.min(startTime.getTime(), endTime.getTime());
|
|
632
|
+
const rangeEndMs = Math.max(startTime.getTime(), endTime.getTime());
|
|
633
|
+
const spanMs = Math.max(1e3, rangeEndMs - rangeStartMs);
|
|
634
|
+
const intervalMs = Math.max(1e3, Math.floor(spanMs / buckets));
|
|
635
|
+
const interval = intervalMsToIntervalString(intervalMs);
|
|
636
|
+
const conditions = buildFilterConditions({
|
|
637
|
+
...params,
|
|
638
|
+
startTime: new Date(rangeStartMs),
|
|
639
|
+
endTime: new Date(rangeEndMs)
|
|
640
|
+
});
|
|
641
|
+
conditions.push("timestamp IS NOT NULL");
|
|
642
|
+
conditions.push("level IS NOT NULL");
|
|
643
|
+
const whereClause = `WHERE ${conditions.join(" AND ")}`;
|
|
644
|
+
const reader = await conn.runAndReadAll(
|
|
645
|
+
`SELECT
|
|
646
|
+
time_bucket(INTERVAL '${interval}', timestamp) AS bucket,
|
|
647
|
+
UPPER(level) AS lvl,
|
|
648
|
+
COUNT(*) AS cnt
|
|
649
|
+
FROM logs
|
|
650
|
+
${whereClause}
|
|
651
|
+
GROUP BY bucket, lvl
|
|
652
|
+
ORDER BY bucket ASC`
|
|
653
|
+
);
|
|
654
|
+
const countsByBucket = /* @__PURE__ */ new Map();
|
|
655
|
+
for (const row of reader.getRowsJS()) {
|
|
656
|
+
const bucketIso = jsToDate(row[0]).toISOString();
|
|
657
|
+
const level = String(row[1]);
|
|
658
|
+
const count = Number(row[2]);
|
|
659
|
+
const current = countsByBucket.get(bucketIso) ?? {};
|
|
660
|
+
current[level] = count;
|
|
661
|
+
countsByBucket.set(bucketIso, current);
|
|
662
|
+
}
|
|
663
|
+
const alignedStart = Math.floor(rangeStartMs / intervalMs) * intervalMs;
|
|
664
|
+
const histogramBuckets = Array.from({ length: buckets }, (_, i) => {
|
|
665
|
+
const timestamp = new Date(alignedStart + i * intervalMs).toISOString();
|
|
666
|
+
return {
|
|
667
|
+
timestamp,
|
|
668
|
+
counts: countsByBucket.get(timestamp) ?? {}
|
|
669
|
+
};
|
|
670
|
+
});
|
|
671
|
+
return {
|
|
672
|
+
buckets: histogramBuckets,
|
|
673
|
+
interval
|
|
674
|
+
};
|
|
675
|
+
}
|
|
676
|
+
// -------------------------------------------------------------------------
|
|
509
677
|
// Custom SQL Execution
|
|
510
678
|
// -------------------------------------------------------------------------
|
|
511
679
|
async executeQuery(sql) {
|
|
@@ -607,6 +775,26 @@ function csvEscape(value) {
|
|
|
607
775
|
}
|
|
608
776
|
return str;
|
|
609
777
|
}
|
|
778
|
+
function intervalMsToIntervalString(intervalMs) {
|
|
779
|
+
const dayMs = 24 * 60 * 60 * 1e3;
|
|
780
|
+
const hourMs = 60 * 60 * 1e3;
|
|
781
|
+
const minuteMs = 60 * 1e3;
|
|
782
|
+
const secondMs = 1e3;
|
|
783
|
+
if (intervalMs % dayMs === 0) {
|
|
784
|
+
const days = Math.max(1, Math.floor(intervalMs / dayMs));
|
|
785
|
+
return `${days} ${days === 1 ? "day" : "days"}`;
|
|
786
|
+
}
|
|
787
|
+
if (intervalMs % hourMs === 0) {
|
|
788
|
+
const hours = Math.max(1, Math.floor(intervalMs / hourMs));
|
|
789
|
+
return `${hours} ${hours === 1 ? "hour" : "hours"}`;
|
|
790
|
+
}
|
|
791
|
+
if (intervalMs % minuteMs === 0) {
|
|
792
|
+
const minutes = Math.max(1, Math.floor(intervalMs / minuteMs));
|
|
793
|
+
return `${minutes} ${minutes === 1 ? "minute" : "minutes"}`;
|
|
794
|
+
}
|
|
795
|
+
const seconds = Math.max(1, Math.floor(intervalMs / secondMs));
|
|
796
|
+
return `${seconds} ${seconds === 1 ? "second" : "seconds"}`;
|
|
797
|
+
}
|
|
610
798
|
function appendNullableVarchar(appender, value) {
|
|
611
799
|
if (value !== null) {
|
|
612
800
|
appender.appendVarchar(value);
|
|
@@ -628,19 +816,45 @@ function appendNullableDouble(appender, value) {
|
|
|
628
816
|
appender.appendNull();
|
|
629
817
|
}
|
|
630
818
|
}
|
|
819
|
+
function buildArrayCondition(column, values, caseInsensitive = false) {
|
|
820
|
+
if (caseInsensitive) {
|
|
821
|
+
const upperValues = values.map((v) => v.toUpperCase());
|
|
822
|
+
if (upperValues.length === 1) {
|
|
823
|
+
return `UPPER(${column}) = '${escapeSql(upperValues[0])}'`;
|
|
824
|
+
}
|
|
825
|
+
const escaped2 = upperValues.map((v) => `'${escapeSql(v)}'`).join(", ");
|
|
826
|
+
return `UPPER(${column}) IN (${escaped2})`;
|
|
827
|
+
}
|
|
828
|
+
if (values.length === 1) {
|
|
829
|
+
return `${column} = '${escapeSql(values[0])}'`;
|
|
830
|
+
}
|
|
831
|
+
const escaped = values.map((v) => `'${escapeSql(v)}'`).join(", ");
|
|
832
|
+
return `${column} IN (${escaped})`;
|
|
833
|
+
}
|
|
631
834
|
function buildFilterConditions(params) {
|
|
632
835
|
const conditions = [];
|
|
633
|
-
if (params.level) {
|
|
634
|
-
conditions.push(
|
|
836
|
+
if (params.level && params.level.length > 0) {
|
|
837
|
+
conditions.push(buildArrayCondition("level", params.level, true));
|
|
635
838
|
}
|
|
636
|
-
if (params.service) {
|
|
637
|
-
conditions.push(
|
|
839
|
+
if (params.service && params.service.length > 0) {
|
|
840
|
+
conditions.push(buildArrayCondition("service", params.service));
|
|
638
841
|
}
|
|
639
|
-
if (params.
|
|
640
|
-
conditions.push(
|
|
842
|
+
if (params.host && params.host.length > 0) {
|
|
843
|
+
conditions.push(buildArrayCondition("host", params.host));
|
|
844
|
+
}
|
|
845
|
+
if (params.source && params.source.length > 0) {
|
|
846
|
+
conditions.push(buildArrayCondition("source", params.source));
|
|
641
847
|
}
|
|
642
848
|
if (params.search) {
|
|
643
|
-
|
|
849
|
+
const parsed = parseSearchQuery(params.search);
|
|
850
|
+
if (parsed.field !== null) {
|
|
851
|
+
conditions.push(`${parsed.field} ILIKE '%${escapeSql(parsed.value)}%'`);
|
|
852
|
+
} else {
|
|
853
|
+
const kw = escapeSql(parsed.value);
|
|
854
|
+
conditions.push(
|
|
855
|
+
`(message ILIKE '%${kw}%' OR service ILIKE '%${kw}%' OR host ILIKE '%${kw}%' OR trace_id ILIKE '%${kw}%' OR CAST(_raw AS VARCHAR) ILIKE '%${kw}%')`
|
|
856
|
+
);
|
|
857
|
+
}
|
|
644
858
|
}
|
|
645
859
|
if (params.startTime) {
|
|
646
860
|
conditions.push(`timestamp >= '${params.startTime.toISOString()}'`);
|
|
@@ -649,9 +863,11 @@ function buildFilterConditions(params) {
|
|
|
649
863
|
conditions.push(`timestamp <= '${params.endTime.toISOString()}'`);
|
|
650
864
|
}
|
|
651
865
|
if (params.jsonFilters) {
|
|
652
|
-
for (const [jsonPath,
|
|
866
|
+
for (const [jsonPath, values] of Object.entries(params.jsonFilters)) {
|
|
653
867
|
const expr = `CAST(json_extract(_raw, '${escapeSql("$." + jsonPath)}') AS VARCHAR)`;
|
|
654
|
-
|
|
868
|
+
if (values.length > 0) {
|
|
869
|
+
conditions.push(buildArrayCondition(expr, values));
|
|
870
|
+
}
|
|
655
871
|
}
|
|
656
872
|
}
|
|
657
873
|
return conditions;
|
|
@@ -661,10 +877,21 @@ function buildWhereClause(params) {
|
|
|
661
877
|
return conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
|
|
662
878
|
}
|
|
663
879
|
function rowToLogEntry(row) {
|
|
880
|
+
let rawParsed;
|
|
881
|
+
if (typeof row[2] === "string") {
|
|
882
|
+
try {
|
|
883
|
+
const parsed = JSON.parse(row[2]);
|
|
884
|
+
rawParsed = typeof parsed === "object" && parsed !== null && !Array.isArray(parsed) ? parsed : { _value: parsed };
|
|
885
|
+
} catch {
|
|
886
|
+
rawParsed = { message: row[2] };
|
|
887
|
+
}
|
|
888
|
+
} else {
|
|
889
|
+
rawParsed = row[2] ?? {};
|
|
890
|
+
}
|
|
664
891
|
return {
|
|
665
892
|
_id: BigInt(row[0]),
|
|
666
893
|
_ingested: jsToDate(row[1]),
|
|
667
|
-
_raw:
|
|
894
|
+
_raw: rawParsed,
|
|
668
895
|
timestamp: row[3] !== null ? jsToDate(row[3]) : null,
|
|
669
896
|
level: row[4],
|
|
670
897
|
message: row[5],
|
|
@@ -675,25 +902,6 @@ function rowToLogEntry(row) {
|
|
|
675
902
|
source: row[10]
|
|
676
903
|
};
|
|
677
904
|
}
|
|
678
|
-
const FIELD_MAPPINGS = {
|
|
679
|
-
timestamp: ["timestamp", "ts", "time", "@timestamp", "datetime"],
|
|
680
|
-
level: ["level", "severity", "loglevel", "lvl", "priority"],
|
|
681
|
-
message: ["message", "msg", "body", "text"],
|
|
682
|
-
service: ["service", "svc", "app", "component", "logger"],
|
|
683
|
-
trace_id: ["trace_id", "traceId", "request_id", "correlation_id"]
|
|
684
|
-
};
|
|
685
|
-
const REVERSE_FIELD_MAP = (() => {
|
|
686
|
-
const map = /* @__PURE__ */ new Map();
|
|
687
|
-
for (const [canonical, aliases] of Object.entries(FIELD_MAPPINGS)) {
|
|
688
|
-
for (const alias of aliases) {
|
|
689
|
-
map.set(alias, canonical);
|
|
690
|
-
}
|
|
691
|
-
}
|
|
692
|
-
return map;
|
|
693
|
-
})();
|
|
694
|
-
function resolveField(fieldName) {
|
|
695
|
-
return REVERSE_FIELD_MAP.get(fieldName) ?? null;
|
|
696
|
-
}
|
|
697
905
|
class Ingester {
|
|
698
906
|
db;
|
|
699
907
|
options;
|
|
@@ -704,6 +912,11 @@ class Ingester {
|
|
|
704
912
|
rl = null;
|
|
705
913
|
stopped = false;
|
|
706
914
|
flushing = null;
|
|
915
|
+
// Sliding window for ingestion rate calculation
|
|
916
|
+
static RATE_WINDOW_MS = 1e4;
|
|
917
|
+
// 10 seconds
|
|
918
|
+
batchRecords = [];
|
|
919
|
+
lastBatchTime = null;
|
|
707
920
|
constructor(db, options) {
|
|
708
921
|
this.db = db;
|
|
709
922
|
this.options = options;
|
|
@@ -763,6 +976,23 @@ class Ingester {
|
|
|
763
976
|
on(event, listener) {
|
|
764
977
|
this.emitter.on(event, listener);
|
|
765
978
|
}
|
|
979
|
+
/**
|
|
980
|
+
* Returns the current ingestion statistics including the smoothed
|
|
981
|
+
* ingestion rate (logs/second) over a 10-second sliding window.
|
|
982
|
+
*/
|
|
983
|
+
getIngestionStats() {
|
|
984
|
+
const now = Date.now();
|
|
985
|
+
const windowStart = now - Ingester.RATE_WINDOW_MS;
|
|
986
|
+
this.batchRecords = this.batchRecords.filter(
|
|
987
|
+
(r) => r.timestamp >= windowStart
|
|
988
|
+
);
|
|
989
|
+
const totalLogs = this.batchRecords.reduce((sum, r) => sum + r.size, 0);
|
|
990
|
+
const ingestionRate = totalLogs / (Ingester.RATE_WINDOW_MS / 1e3);
|
|
991
|
+
return {
|
|
992
|
+
ingestionRate,
|
|
993
|
+
lastBatchTime: this.lastBatchTime
|
|
994
|
+
};
|
|
995
|
+
}
|
|
766
996
|
// -------------------------------------------------------------------------
|
|
767
997
|
// Line processing
|
|
768
998
|
// -------------------------------------------------------------------------
|
|
@@ -770,15 +1000,18 @@ class Ingester {
|
|
|
770
1000
|
const trimmed = line.trim();
|
|
771
1001
|
if (trimmed === "") return;
|
|
772
1002
|
let parsed;
|
|
1003
|
+
let raw;
|
|
773
1004
|
try {
|
|
774
1005
|
parsed = JSON.parse(trimmed);
|
|
1006
|
+
raw = trimmed;
|
|
775
1007
|
} catch {
|
|
776
|
-
|
|
1008
|
+
parsed = { message: trimmed, level: "INFO", timestamp: (/* @__PURE__ */ new Date()).toISOString() };
|
|
1009
|
+
raw = JSON.stringify(parsed);
|
|
777
1010
|
}
|
|
778
1011
|
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
|
|
779
1012
|
return;
|
|
780
1013
|
}
|
|
781
|
-
const normalized = this.normalize(parsed,
|
|
1014
|
+
const normalized = this.normalize(parsed, raw);
|
|
782
1015
|
this.buffer.push(normalized);
|
|
783
1016
|
if (this.buffer.length >= this.options.batchSize) {
|
|
784
1017
|
this.flushBuffer();
|
|
@@ -843,6 +1076,9 @@ class Ingester {
|
|
|
843
1076
|
try {
|
|
844
1077
|
await this.db.insertBatch(batch);
|
|
845
1078
|
await this.db.evictOldRows(this.options.maxRows);
|
|
1079
|
+
const now = Date.now();
|
|
1080
|
+
this.batchRecords.push({ timestamp: now, size: batch.length });
|
|
1081
|
+
this.lastBatchTime = new Date(now);
|
|
846
1082
|
this.emitter.emit("batch", batch);
|
|
847
1083
|
} catch (err) {
|
|
848
1084
|
console.error("[ingester] doFlush error:", err);
|
|
@@ -864,13 +1100,16 @@ class Ingester {
|
|
|
864
1100
|
}
|
|
865
1101
|
}
|
|
866
1102
|
function matchesFilter(log, filter) {
|
|
867
|
-
if (filter.level
|
|
1103
|
+
if (filter.level && filter.level.length > 0 && !filter.level.map((l) => l.toUpperCase()).includes(log.level?.toUpperCase())) {
|
|
1104
|
+
return false;
|
|
1105
|
+
}
|
|
1106
|
+
if (filter.service && filter.service.length > 0 && !filter.service.includes(log.service ?? "")) {
|
|
868
1107
|
return false;
|
|
869
1108
|
}
|
|
870
|
-
if (filter.
|
|
1109
|
+
if (filter.host && filter.host.length > 0 && !filter.host.includes(log.host ?? "")) {
|
|
871
1110
|
return false;
|
|
872
1111
|
}
|
|
873
|
-
if (filter.source
|
|
1112
|
+
if (filter.source && filter.source.length > 0 && !filter.source.includes(log.source)) {
|
|
874
1113
|
return false;
|
|
875
1114
|
}
|
|
876
1115
|
if (filter.search !== void 0 && filter.search !== "") {
|
|
@@ -910,7 +1149,8 @@ function serializeStats(stats) {
|
|
|
910
1149
|
timeRange: {
|
|
911
1150
|
min: stats.timeRange.min?.toISOString() ?? null,
|
|
912
1151
|
max: stats.timeRange.max?.toISOString() ?? null
|
|
913
|
-
}
|
|
1152
|
+
},
|
|
1153
|
+
ingestionRate: stats.ingestionRate
|
|
914
1154
|
};
|
|
915
1155
|
}
|
|
916
1156
|
class WSHandler {
|
|
@@ -940,6 +1180,8 @@ class WSHandler {
|
|
|
940
1180
|
}
|
|
941
1181
|
}
|
|
942
1182
|
if (stats) {
|
|
1183
|
+
const ingestionStats = ingester.getIngestionStats();
|
|
1184
|
+
stats.ingestionRate = ingestionStats.ingestionRate;
|
|
943
1185
|
this.broadcast([...logs], stats);
|
|
944
1186
|
}
|
|
945
1187
|
});
|
|
@@ -976,14 +1218,25 @@ class WSHandler {
|
|
|
976
1218
|
}
|
|
977
1219
|
const filterObj = msg.filter;
|
|
978
1220
|
const filter = {};
|
|
979
|
-
if (
|
|
980
|
-
filter.level = filterObj.level;
|
|
1221
|
+
if (Array.isArray(filterObj.level)) {
|
|
1222
|
+
filter.level = filterObj.level.filter((v) => typeof v === "string");
|
|
1223
|
+
} else if (typeof filterObj.level === "string") {
|
|
1224
|
+
filter.level = [filterObj.level];
|
|
981
1225
|
}
|
|
982
|
-
if (
|
|
983
|
-
filter.service = filterObj.service;
|
|
1226
|
+
if (Array.isArray(filterObj.service)) {
|
|
1227
|
+
filter.service = filterObj.service.filter((v) => typeof v === "string");
|
|
1228
|
+
} else if (typeof filterObj.service === "string") {
|
|
1229
|
+
filter.service = [filterObj.service];
|
|
984
1230
|
}
|
|
985
|
-
if (
|
|
986
|
-
filter.
|
|
1231
|
+
if (Array.isArray(filterObj.host)) {
|
|
1232
|
+
filter.host = filterObj.host.filter((v) => typeof v === "string");
|
|
1233
|
+
} else if (typeof filterObj.host === "string") {
|
|
1234
|
+
filter.host = [filterObj.host];
|
|
1235
|
+
}
|
|
1236
|
+
if (Array.isArray(filterObj.source)) {
|
|
1237
|
+
filter.source = filterObj.source.filter((v) => typeof v === "string");
|
|
1238
|
+
} else if (typeof filterObj.source === "string") {
|
|
1239
|
+
filter.source = [filterObj.source];
|
|
987
1240
|
}
|
|
988
1241
|
if (typeof filterObj.search === "string") {
|
|
989
1242
|
filter.search = filterObj.search;
|
|
@@ -1037,6 +1290,7 @@ function parseRelayArgs(args) {
|
|
|
1037
1290
|
service: { type: "string", short: "s" },
|
|
1038
1291
|
"batch-size": { type: "string" },
|
|
1039
1292
|
interval: { type: "string" },
|
|
1293
|
+
"max-retries": { type: "string" },
|
|
1040
1294
|
help: { type: "boolean", short: "h" }
|
|
1041
1295
|
},
|
|
1042
1296
|
strict: true
|
|
@@ -1046,6 +1300,7 @@ function parseRelayArgs(args) {
|
|
|
1046
1300
|
service: values.service,
|
|
1047
1301
|
batchSize: values["batch-size"] !== void 0 ? parseInt(values["batch-size"], 10) : 100,
|
|
1048
1302
|
intervalMs: values.interval !== void 0 ? parseInt(values.interval, 10) : 500,
|
|
1303
|
+
maxRetries: values["max-retries"] !== void 0 ? parseInt(values["max-retries"], 10) : 3,
|
|
1049
1304
|
help: values.help ?? false
|
|
1050
1305
|
};
|
|
1051
1306
|
}
|
|
@@ -1059,6 +1314,7 @@ Options:
|
|
|
1059
1314
|
-s, --service <name> Service name to inject into each log entry
|
|
1060
1315
|
--batch-size <n> Lines per HTTP batch (default: 100)
|
|
1061
1316
|
--interval <ms> Flush interval in milliseconds (default: 500)
|
|
1317
|
+
--max-retries <n> Max retry attempts on transient failure (default: 3)
|
|
1062
1318
|
-h, --help Show this help message
|
|
1063
1319
|
|
|
1064
1320
|
Examples:
|
|
@@ -1066,6 +1322,75 @@ Examples:
|
|
|
1066
1322
|
cat app.log | lduck relay --url http://lduck:9090 --service backend
|
|
1067
1323
|
docker logs -f myapp | lduck relay -s myapp -u http://localhost:8080
|
|
1068
1324
|
`.trimStart();
|
|
1325
|
+
function parseLine(line, service) {
|
|
1326
|
+
const trimmed = line.trim();
|
|
1327
|
+
if (trimmed === "") return null;
|
|
1328
|
+
let parsed;
|
|
1329
|
+
try {
|
|
1330
|
+
parsed = JSON.parse(trimmed);
|
|
1331
|
+
} catch {
|
|
1332
|
+
parsed = { message: trimmed, level: "INFO", timestamp: (/* @__PURE__ */ new Date()).toISOString() };
|
|
1333
|
+
}
|
|
1334
|
+
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
|
|
1335
|
+
return null;
|
|
1336
|
+
}
|
|
1337
|
+
if (service !== void 0 && !("service" in parsed)) {
|
|
1338
|
+
parsed.service = service;
|
|
1339
|
+
}
|
|
1340
|
+
return parsed;
|
|
1341
|
+
}
|
|
1342
|
+
function computeBackoffDelay(attempt, baseDelay, maxDelay) {
|
|
1343
|
+
return Math.min(baseDelay * 2 ** attempt, maxDelay);
|
|
1344
|
+
}
|
|
1345
|
+
function accumulateFlushResult(summary, result) {
|
|
1346
|
+
return {
|
|
1347
|
+
totalSent: summary.totalSent + result.accepted,
|
|
1348
|
+
totalErrors: summary.totalErrors + result.errors,
|
|
1349
|
+
totalRetries: summary.totalRetries + result.retries
|
|
1350
|
+
};
|
|
1351
|
+
}
|
|
1352
|
+
function formatRelaySummary(summary) {
|
|
1353
|
+
return `[relay] Done. Sent: ${summary.totalSent}, Errors: ${summary.totalErrors}, Retries: ${summary.totalRetries}`;
|
|
1354
|
+
}
|
|
1355
|
+
const DEFAULT_BASE_DELAY = 100;
|
|
1356
|
+
const DEFAULT_MAX_DELAY = 3e4;
|
|
1357
|
+
function isRetryableStatus(status) {
|
|
1358
|
+
return status >= 500;
|
|
1359
|
+
}
|
|
1360
|
+
async function flushWithRetry(batch, ingestUrl, maxRetries, fetchFn = globalThis.fetch, backoffOptions) {
|
|
1361
|
+
const baseDelay = DEFAULT_BASE_DELAY;
|
|
1362
|
+
const maxDelay = DEFAULT_MAX_DELAY;
|
|
1363
|
+
let retries = 0;
|
|
1364
|
+
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
|
1365
|
+
if (attempt > 0) {
|
|
1366
|
+
const delay = computeBackoffDelay(attempt - 1, baseDelay, maxDelay);
|
|
1367
|
+
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
1368
|
+
retries++;
|
|
1369
|
+
}
|
|
1370
|
+
try {
|
|
1371
|
+
const res = await fetchFn(ingestUrl, {
|
|
1372
|
+
method: "POST",
|
|
1373
|
+
headers: { "Content-Type": "application/json" },
|
|
1374
|
+
body: JSON.stringify(batch)
|
|
1375
|
+
});
|
|
1376
|
+
if (res.ok) {
|
|
1377
|
+
const result = await res.json();
|
|
1378
|
+
return { accepted: result.accepted, errors: 0, retries };
|
|
1379
|
+
}
|
|
1380
|
+
if (!isRetryableStatus(res.status)) {
|
|
1381
|
+
const text2 = await res.text();
|
|
1382
|
+
console.error(`[relay] HTTP ${res.status}: ${text2}`);
|
|
1383
|
+
return { accepted: 0, errors: batch.length, retries: 0 };
|
|
1384
|
+
}
|
|
1385
|
+
const text = await res.text();
|
|
1386
|
+
console.error(`[relay] HTTP ${res.status}: ${text} (attempt ${attempt + 1}/${maxRetries + 1})`);
|
|
1387
|
+
} catch (err) {
|
|
1388
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
1389
|
+
console.error(`[relay] Send failed: ${message} (attempt ${attempt + 1}/${maxRetries + 1})`);
|
|
1390
|
+
}
|
|
1391
|
+
}
|
|
1392
|
+
return { accepted: 0, errors: batch.length, retries };
|
|
1393
|
+
}
|
|
1069
1394
|
async function runRelay(args) {
|
|
1070
1395
|
const opts = parseRelayArgs(args);
|
|
1071
1396
|
if (opts.help) {
|
|
@@ -1075,32 +1400,14 @@ async function runRelay(args) {
|
|
|
1075
1400
|
const ingestUrl = opts.url.replace(/\/$/, "") + "/api/ingest";
|
|
1076
1401
|
let buffer = [];
|
|
1077
1402
|
let flushTimer = null;
|
|
1078
|
-
let
|
|
1079
|
-
let totalErrors = 0;
|
|
1403
|
+
let summary = { totalSent: 0, totalErrors: 0, totalRetries: 0 };
|
|
1080
1404
|
let stdinClosed = false;
|
|
1081
1405
|
async function flush() {
|
|
1082
1406
|
if (buffer.length === 0) return;
|
|
1083
1407
|
const batch = buffer;
|
|
1084
1408
|
buffer = [];
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
method: "POST",
|
|
1088
|
-
headers: { "Content-Type": "application/json" },
|
|
1089
|
-
body: JSON.stringify(batch)
|
|
1090
|
-
});
|
|
1091
|
-
if (!res.ok) {
|
|
1092
|
-
const text = await res.text();
|
|
1093
|
-
console.error(`[relay] HTTP ${res.status}: ${text}`);
|
|
1094
|
-
totalErrors += batch.length;
|
|
1095
|
-
} else {
|
|
1096
|
-
const result = await res.json();
|
|
1097
|
-
totalSent += result.accepted;
|
|
1098
|
-
}
|
|
1099
|
-
} catch (err) {
|
|
1100
|
-
const message = err instanceof Error ? err.message : String(err);
|
|
1101
|
-
console.error(`[relay] Send failed: ${message}`);
|
|
1102
|
-
totalErrors += batch.length;
|
|
1103
|
-
}
|
|
1409
|
+
const result = await flushWithRetry(batch, ingestUrl, opts.maxRetries);
|
|
1410
|
+
summary = accumulateFlushResult(summary, result);
|
|
1104
1411
|
}
|
|
1105
1412
|
function resetFlushTimer() {
|
|
1106
1413
|
if (flushTimer !== null) {
|
|
@@ -1121,20 +1428,8 @@ async function runRelay(args) {
|
|
|
1121
1428
|
console.error(`[relay] Relaying to ${ingestUrl}${opts.service ? ` (service: ${opts.service})` : ""}`);
|
|
1122
1429
|
resetFlushTimer();
|
|
1123
1430
|
rl.on("line", (line) => {
|
|
1124
|
-
const
|
|
1125
|
-
if (
|
|
1126
|
-
let parsed;
|
|
1127
|
-
try {
|
|
1128
|
-
parsed = JSON.parse(trimmed);
|
|
1129
|
-
} catch {
|
|
1130
|
-
return;
|
|
1131
|
-
}
|
|
1132
|
-
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
|
|
1133
|
-
return;
|
|
1134
|
-
}
|
|
1135
|
-
if (opts.service !== void 0 && !("service" in parsed)) {
|
|
1136
|
-
parsed.service = opts.service;
|
|
1137
|
-
}
|
|
1431
|
+
const parsed = parseLine(line, opts.service);
|
|
1432
|
+
if (parsed === null) return;
|
|
1138
1433
|
buffer.push(parsed);
|
|
1139
1434
|
if (buffer.length >= opts.batchSize) {
|
|
1140
1435
|
flush();
|
|
@@ -1148,11 +1443,7 @@ async function runRelay(args) {
|
|
|
1148
1443
|
flushTimer = null;
|
|
1149
1444
|
}
|
|
1150
1445
|
await flush();
|
|
1151
|
-
|
|
1152
|
-
console.error(
|
|
1153
|
-
`[relay] Done. Sent: ${totalSent}, Errors: ${totalErrors}`
|
|
1154
|
-
);
|
|
1155
|
-
}
|
|
1446
|
+
console.error(formatRelaySummary(summary));
|
|
1156
1447
|
resolve();
|
|
1157
1448
|
});
|
|
1158
1449
|
});
|