lduck 0.0.2 → 0.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -53,9 +53,64 @@ function createApp(scriptSrc, setup) {
53
53
  });
54
54
  return app;
55
55
  }
56
+ const LOG_LEVELS = [
57
+ "DEBUG",
58
+ "INFO",
59
+ "WARN",
60
+ "ERROR",
61
+ "FATAL"
62
+ ];
63
+ const FIELD_MAPPINGS = {
64
+ timestamp: ["timestamp", "ts", "time", "@timestamp", "datetime"],
65
+ level: ["level", "severity", "loglevel", "lvl", "priority"],
66
+ message: ["message", "msg", "body", "text"],
67
+ service: ["service", "svc", "app", "component", "logger"],
68
+ trace_id: ["trace_id", "traceId", "request_id", "correlation_id"]
69
+ };
70
+ const REVERSE_FIELD_MAP = (() => {
71
+ const map = /* @__PURE__ */ new Map();
72
+ for (const [canonical, aliases] of Object.entries(FIELD_MAPPINGS)) {
73
+ for (const alias of aliases) {
74
+ map.set(alias, canonical);
75
+ }
76
+ }
77
+ return map;
78
+ })();
79
+ function resolveField(fieldName) {
80
+ return REVERSE_FIELD_MAP.get(fieldName) ?? null;
81
+ }
82
+ function splitLevels(csv) {
83
+ if (!csv) return void 0;
84
+ const levels = csv.split(",").filter((v) => LOG_LEVELS.includes(v));
85
+ return levels.length > 0 ? levels : void 0;
86
+ }
87
+ function splitStrings(csv) {
88
+ if (!csv) return void 0;
89
+ const values = csv.split(",").filter(Boolean);
90
+ return values.length > 0 ? values : void 0;
91
+ }
92
+ function parseJsonFilters(raw) {
93
+ if (!raw) return void 0;
94
+ try {
95
+ const parsed = JSON.parse(raw);
96
+ if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) return void 0;
97
+ const result = {};
98
+ for (const [key, value] of Object.entries(parsed)) {
99
+ if (Array.isArray(value)) {
100
+ result[key] = value.map(String);
101
+ } else if (typeof value === "string") {
102
+ result[key] = [value];
103
+ }
104
+ }
105
+ return Object.keys(result).length > 0 ? result : void 0;
106
+ } catch {
107
+ return void 0;
108
+ }
109
+ }
56
110
  const logQuerySchema = z.object({
57
- level: z.enum(["DEBUG", "INFO", "WARN", "ERROR", "FATAL"]).optional(),
111
+ level: z.string().optional(),
58
112
  service: z.string().optional(),
113
+ host: z.string().optional(),
59
114
  source: z.string().optional(),
60
115
  search: z.string().optional(),
61
116
  startTime: z.coerce.date().optional(),
@@ -73,7 +128,7 @@ const sqlQuerySchema = z.object({
73
128
  });
74
129
  const exportSchema = z.object({
75
130
  format: z.enum(["csv", "json"]),
76
- level: z.enum(["DEBUG", "INFO", "WARN", "ERROR", "FATAL"]).optional(),
131
+ level: z.string().optional(),
77
132
  service: z.string().optional(),
78
133
  source: z.string().optional(),
79
134
  search: z.string().optional(),
@@ -87,14 +142,26 @@ const ingestBodySchema = z.union([
87
142
  const facetQuerySchema = z.object({
88
143
  field: z.string(),
89
144
  jsonPath: z.string().optional(),
90
- level: z.enum(["DEBUG", "INFO", "WARN", "ERROR", "FATAL"]).optional(),
145
+ level: z.string().optional(),
91
146
  service: z.string().optional(),
147
+ host: z.string().optional(),
92
148
  source: z.string().optional(),
93
149
  search: z.string().optional(),
94
150
  startTime: z.coerce.date().optional(),
95
151
  endTime: z.coerce.date().optional(),
96
152
  jsonFilters: z.string().optional()
97
153
  });
154
+ const histogramQuerySchema = z.object({
155
+ buckets: z.coerce.number().int().min(1).max(360).default(30),
156
+ startTime: z.coerce.date().optional(),
157
+ endTime: z.coerce.date().optional(),
158
+ level: z.string().optional(),
159
+ service: z.string().optional(),
160
+ host: z.string().optional(),
161
+ source: z.string().optional(),
162
+ search: z.string().optional(),
163
+ jsonFilters: z.string().optional()
164
+ });
98
165
  function validationHook(result, c) {
99
166
  if (!result.success) {
100
167
  return c.json(
@@ -132,7 +199,8 @@ function serializeStats$1(stats) {
132
199
  timeRange: {
133
200
  min: stats.timeRange.min?.toISOString() ?? null,
134
201
  max: stats.timeRange.max?.toISOString() ?? null
135
- }
202
+ },
203
+ ingestionRate: stats.ingestionRate
136
204
  };
137
205
  }
138
206
  function createApiApp(db, ingester) {
@@ -159,6 +227,10 @@ function createApiApp(db, ingester) {
159
227
  try {
160
228
  const { source } = c.req.valid("query");
161
229
  const stats = await db.getStats(source ? { source } : void 0);
230
+ if (ingester) {
231
+ const ingestionStats = ingester.getIngestionStats();
232
+ stats.ingestionRate = ingestionStats.ingestionRate;
233
+ }
162
234
  return c.json(serializeStats$1(stats));
163
235
  } catch (err) {
164
236
  return c.json(
@@ -174,24 +246,18 @@ function createApiApp(db, ingester) {
174
246
  }).get("/logs", zValidator("query", logQuerySchema, validationHook), async (c) => {
175
247
  try {
176
248
  const params = c.req.valid("query");
177
- let jsonFilters;
178
- if (params.jsonFilters) {
179
- try {
180
- jsonFilters = JSON.parse(params.jsonFilters);
181
- } catch {
182
- }
183
- }
184
249
  const queryParams = {
185
- level: params.level,
186
- service: params.service,
187
- source: params.source,
250
+ level: splitLevels(params.level),
251
+ service: splitStrings(params.service),
252
+ host: splitStrings(params.host),
253
+ source: splitStrings(params.source),
188
254
  search: params.search,
189
255
  startTime: params.startTime,
190
256
  endTime: params.endTime,
191
257
  limit: params.limit,
192
258
  offset: params.offset,
193
259
  order: params.order,
194
- jsonFilters
260
+ jsonFilters: parseJsonFilters(params.jsonFilters)
195
261
  };
196
262
  const result = await db.queryLogs(queryParams);
197
263
  return c.json({
@@ -209,6 +275,32 @@ function createApiApp(db, ingester) {
209
275
  500
210
276
  );
211
277
  }
278
+ }).get("/histogram", zValidator("query", histogramQuerySchema, validationHook), async (c) => {
279
+ try {
280
+ const params = c.req.valid("query");
281
+ const result = await db.getHistogram({
282
+ buckets: params.buckets,
283
+ startTime: params.startTime,
284
+ endTime: params.endTime,
285
+ level: splitLevels(params.level),
286
+ service: splitStrings(params.service),
287
+ host: splitStrings(params.host),
288
+ source: splitStrings(params.source),
289
+ search: params.search,
290
+ jsonFilters: parseJsonFilters(params.jsonFilters)
291
+ });
292
+ return c.json(result);
293
+ } catch (err) {
294
+ return c.json(
295
+ {
296
+ error: {
297
+ code: "QUERY_ERROR",
298
+ message: err instanceof Error ? err.message : "Unknown error"
299
+ }
300
+ },
301
+ 500
302
+ );
303
+ }
212
304
  }).post("/query", zValidator("json", sqlQuerySchema, validationHook), async (c) => {
213
305
  try {
214
306
  const { sql } = c.req.valid("json");
@@ -241,9 +333,9 @@ function createApiApp(db, ingester) {
241
333
  try {
242
334
  const params = c.req.valid("json");
243
335
  const queryParams = {
244
- level: params.level,
245
- service: params.service,
246
- source: params.source,
336
+ level: splitLevels(params.level),
337
+ service: splitStrings(params.service),
338
+ source: splitStrings(params.source),
247
339
  search: params.search,
248
340
  startTime: params.startTime,
249
341
  endTime: params.endTime,
@@ -293,21 +385,15 @@ function createApiApp(db, ingester) {
293
385
  }).get("/facets", zValidator("query", facetQuerySchema, validationHook), async (c) => {
294
386
  try {
295
387
  const params = c.req.valid("query");
296
- let jsonFilters;
297
- if (params.jsonFilters) {
298
- try {
299
- jsonFilters = JSON.parse(params.jsonFilters);
300
- } catch {
301
- }
302
- }
303
388
  const filters = {
304
- level: params.level,
305
- service: params.service,
306
- source: params.source,
389
+ level: splitLevels(params.level),
390
+ service: splitStrings(params.service),
391
+ host: splitStrings(params.host),
392
+ source: splitStrings(params.source),
307
393
  search: params.search,
308
394
  startTime: params.startTime,
309
395
  endTime: params.endTime,
310
- jsonFilters
396
+ jsonFilters: parseJsonFilters(params.jsonFilters)
311
397
  };
312
398
  const distribution = await db.getFacetDistribution(
313
399
  params.field,
@@ -333,6 +419,31 @@ new Hono().basePath("/api").get(
333
419
  "/health",
334
420
  (c) => c.json({ status: "ok", uptime: 0 })
335
421
  );
422
+ const SEARCHABLE_FIELDS = /* @__PURE__ */ new Set([
423
+ "message",
424
+ "service",
425
+ "host",
426
+ "trace_id",
427
+ "level"
428
+ ]);
429
+ function parseSearchQuery(search) {
430
+ const colonIndex = search.indexOf(":");
431
+ if (colonIndex <= 0) {
432
+ return { field: null, value: search };
433
+ }
434
+ const field = search.substring(0, colonIndex);
435
+ const value = search.substring(colonIndex + 1);
436
+ if (field.includes(" ")) {
437
+ return { field: null, value: search };
438
+ }
439
+ if (value === "") {
440
+ return { field: null, value: search };
441
+ }
442
+ if (!SEARCHABLE_FIELDS.has(field)) {
443
+ return { field: null, value: search };
444
+ }
445
+ return { field, value };
446
+ }
336
447
  const CREATE_TABLE_SQL = `
337
448
  CREATE TABLE IF NOT EXISTS logs (
338
449
  _id BIGINT PRIMARY KEY,
@@ -457,12 +568,13 @@ class LogDatabase {
457
568
  total: 0,
458
569
  byLevel: {},
459
570
  errorRate: 0,
460
- timeRange: { min: null, max: null }
571
+ timeRange: { min: null, max: null },
572
+ ingestionRate: 0
461
573
  };
462
574
  }
463
575
  const levelWhereClause = whereClause ? `${whereClause} AND level IS NOT NULL` : "WHERE level IS NOT NULL";
464
576
  const levelReader = await conn.runAndReadAll(
465
- `SELECT level, COUNT(*) as cnt FROM logs ${levelWhereClause} GROUP BY level ORDER BY cnt DESC`
577
+ `SELECT UPPER(level) as lvl, COUNT(*) as cnt FROM logs ${levelWhereClause} GROUP BY lvl ORDER BY cnt DESC`
466
578
  );
467
579
  const byLevel = {};
468
580
  for (const row of levelReader.getRowsJS()) {
@@ -481,7 +593,8 @@ class LogDatabase {
481
593
  timeRange: {
482
594
  min: timeRow[0] !== null ? jsToDate(timeRow[0]) : null,
483
595
  max: timeRow[1] !== null ? jsToDate(timeRow[1]) : null
484
- }
596
+ },
597
+ ingestionRate: 0
485
598
  };
486
599
  }
487
600
  // -------------------------------------------------------------------------
@@ -490,11 +603,13 @@ class LogDatabase {
490
603
  async getFacetDistribution(field, jsonPath, filters) {
491
604
  const conn = this.getConnection();
492
605
  const columnExpr = jsonPath !== null ? `CAST(json_extract(_raw, '${escapeSql("$." + jsonPath)}') AS VARCHAR)` : `"${field}"`;
606
+ const isLevelField = field === "level" && jsonPath === null;
607
+ const selectExpr = isLevelField ? `UPPER(${columnExpr})` : columnExpr;
493
608
  const conditions = buildFilterConditions(filters);
494
609
  conditions.push(`${columnExpr} IS NOT NULL`);
495
610
  const whereClause = `WHERE ${conditions.join(" AND ")}`;
496
611
  const reader = await conn.runAndReadAll(
497
- `SELECT ${columnExpr} as val, COUNT(*) as cnt FROM logs ${whereClause} GROUP BY val ORDER BY cnt DESC, val ASC`
612
+ `SELECT ${selectExpr} as val, COUNT(*) as cnt FROM logs ${whereClause} GROUP BY val ORDER BY cnt DESC, val ASC`
498
613
  );
499
614
  const values = reader.getRowsJS().map((row) => {
500
615
  let value = String(row[0]);
@@ -506,6 +621,59 @@ class LogDatabase {
506
621
  return { field, values };
507
622
  }
508
623
  // -------------------------------------------------------------------------
624
+ // Histogram
625
+ // -------------------------------------------------------------------------
626
+ async getHistogram(params) {
627
+ const conn = this.getConnection();
628
+ const buckets = Math.min(360, Math.max(1, Math.floor(params.buckets ?? 30)));
629
+ const endTime = params.endTime ?? /* @__PURE__ */ new Date();
630
+ const startTime = params.startTime ?? new Date(endTime.getTime() - 60 * 60 * 1e3);
631
+ const rangeStartMs = Math.min(startTime.getTime(), endTime.getTime());
632
+ const rangeEndMs = Math.max(startTime.getTime(), endTime.getTime());
633
+ const spanMs = Math.max(1e3, rangeEndMs - rangeStartMs);
634
+ const intervalMs = Math.max(1e3, Math.floor(spanMs / buckets));
635
+ const interval = intervalMsToIntervalString(intervalMs);
636
+ const conditions = buildFilterConditions({
637
+ ...params,
638
+ startTime: new Date(rangeStartMs),
639
+ endTime: new Date(rangeEndMs)
640
+ });
641
+ conditions.push("timestamp IS NOT NULL");
642
+ conditions.push("level IS NOT NULL");
643
+ const whereClause = `WHERE ${conditions.join(" AND ")}`;
644
+ const reader = await conn.runAndReadAll(
645
+ `SELECT
646
+ time_bucket(INTERVAL '${interval}', timestamp) AS bucket,
647
+ UPPER(level) AS lvl,
648
+ COUNT(*) AS cnt
649
+ FROM logs
650
+ ${whereClause}
651
+ GROUP BY bucket, lvl
652
+ ORDER BY bucket ASC`
653
+ );
654
+ const countsByBucket = /* @__PURE__ */ new Map();
655
+ for (const row of reader.getRowsJS()) {
656
+ const bucketIso = jsToDate(row[0]).toISOString();
657
+ const level = String(row[1]);
658
+ const count = Number(row[2]);
659
+ const current = countsByBucket.get(bucketIso) ?? {};
660
+ current[level] = count;
661
+ countsByBucket.set(bucketIso, current);
662
+ }
663
+ const alignedStart = Math.floor(rangeStartMs / intervalMs) * intervalMs;
664
+ const histogramBuckets = Array.from({ length: buckets }, (_, i) => {
665
+ const timestamp = new Date(alignedStart + i * intervalMs).toISOString();
666
+ return {
667
+ timestamp,
668
+ counts: countsByBucket.get(timestamp) ?? {}
669
+ };
670
+ });
671
+ return {
672
+ buckets: histogramBuckets,
673
+ interval
674
+ };
675
+ }
676
+ // -------------------------------------------------------------------------
509
677
  // Custom SQL Execution
510
678
  // -------------------------------------------------------------------------
511
679
  async executeQuery(sql) {
@@ -607,6 +775,26 @@ function csvEscape(value) {
607
775
  }
608
776
  return str;
609
777
  }
778
+ function intervalMsToIntervalString(intervalMs) {
779
+ const dayMs = 24 * 60 * 60 * 1e3;
780
+ const hourMs = 60 * 60 * 1e3;
781
+ const minuteMs = 60 * 1e3;
782
+ const secondMs = 1e3;
783
+ if (intervalMs % dayMs === 0) {
784
+ const days = Math.max(1, Math.floor(intervalMs / dayMs));
785
+ return `${days} ${days === 1 ? "day" : "days"}`;
786
+ }
787
+ if (intervalMs % hourMs === 0) {
788
+ const hours = Math.max(1, Math.floor(intervalMs / hourMs));
789
+ return `${hours} ${hours === 1 ? "hour" : "hours"}`;
790
+ }
791
+ if (intervalMs % minuteMs === 0) {
792
+ const minutes = Math.max(1, Math.floor(intervalMs / minuteMs));
793
+ return `${minutes} ${minutes === 1 ? "minute" : "minutes"}`;
794
+ }
795
+ const seconds = Math.max(1, Math.floor(intervalMs / secondMs));
796
+ return `${seconds} ${seconds === 1 ? "second" : "seconds"}`;
797
+ }
610
798
  function appendNullableVarchar(appender, value) {
611
799
  if (value !== null) {
612
800
  appender.appendVarchar(value);
@@ -628,19 +816,45 @@ function appendNullableDouble(appender, value) {
628
816
  appender.appendNull();
629
817
  }
630
818
  }
819
+ function buildArrayCondition(column, values, caseInsensitive = false) {
820
+ if (caseInsensitive) {
821
+ const upperValues = values.map((v) => v.toUpperCase());
822
+ if (upperValues.length === 1) {
823
+ return `UPPER(${column}) = '${escapeSql(upperValues[0])}'`;
824
+ }
825
+ const escaped2 = upperValues.map((v) => `'${escapeSql(v)}'`).join(", ");
826
+ return `UPPER(${column}) IN (${escaped2})`;
827
+ }
828
+ if (values.length === 1) {
829
+ return `${column} = '${escapeSql(values[0])}'`;
830
+ }
831
+ const escaped = values.map((v) => `'${escapeSql(v)}'`).join(", ");
832
+ return `${column} IN (${escaped})`;
833
+ }
631
834
  function buildFilterConditions(params) {
632
835
  const conditions = [];
633
- if (params.level) {
634
- conditions.push(`level = '${escapeSql(params.level)}'`);
836
+ if (params.level && params.level.length > 0) {
837
+ conditions.push(buildArrayCondition("level", params.level, true));
635
838
  }
636
- if (params.service) {
637
- conditions.push(`service = '${escapeSql(params.service)}'`);
839
+ if (params.service && params.service.length > 0) {
840
+ conditions.push(buildArrayCondition("service", params.service));
638
841
  }
639
- if (params.source) {
640
- conditions.push(`source = '${escapeSql(params.source)}'`);
842
+ if (params.host && params.host.length > 0) {
843
+ conditions.push(buildArrayCondition("host", params.host));
844
+ }
845
+ if (params.source && params.source.length > 0) {
846
+ conditions.push(buildArrayCondition("source", params.source));
641
847
  }
642
848
  if (params.search) {
643
- conditions.push(`message ILIKE '%${escapeSql(params.search)}%'`);
849
+ const parsed = parseSearchQuery(params.search);
850
+ if (parsed.field !== null) {
851
+ conditions.push(`${parsed.field} ILIKE '%${escapeSql(parsed.value)}%'`);
852
+ } else {
853
+ const kw = escapeSql(parsed.value);
854
+ conditions.push(
855
+ `(message ILIKE '%${kw}%' OR service ILIKE '%${kw}%' OR host ILIKE '%${kw}%' OR trace_id ILIKE '%${kw}%' OR CAST(_raw AS VARCHAR) ILIKE '%${kw}%')`
856
+ );
857
+ }
644
858
  }
645
859
  if (params.startTime) {
646
860
  conditions.push(`timestamp >= '${params.startTime.toISOString()}'`);
@@ -649,9 +863,11 @@ function buildFilterConditions(params) {
649
863
  conditions.push(`timestamp <= '${params.endTime.toISOString()}'`);
650
864
  }
651
865
  if (params.jsonFilters) {
652
- for (const [jsonPath, value] of Object.entries(params.jsonFilters)) {
866
+ for (const [jsonPath, values] of Object.entries(params.jsonFilters)) {
653
867
  const expr = `CAST(json_extract(_raw, '${escapeSql("$." + jsonPath)}') AS VARCHAR)`;
654
- conditions.push(`${expr} = '${escapeSql(value)}'`);
868
+ if (values.length > 0) {
869
+ conditions.push(buildArrayCondition(expr, values));
870
+ }
655
871
  }
656
872
  }
657
873
  return conditions;
@@ -661,10 +877,21 @@ function buildWhereClause(params) {
661
877
  return conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
662
878
  }
663
879
  function rowToLogEntry(row) {
880
+ let rawParsed;
881
+ if (typeof row[2] === "string") {
882
+ try {
883
+ const parsed = JSON.parse(row[2]);
884
+ rawParsed = typeof parsed === "object" && parsed !== null && !Array.isArray(parsed) ? parsed : { _value: parsed };
885
+ } catch {
886
+ rawParsed = { message: row[2] };
887
+ }
888
+ } else {
889
+ rawParsed = row[2] ?? {};
890
+ }
664
891
  return {
665
892
  _id: BigInt(row[0]),
666
893
  _ingested: jsToDate(row[1]),
667
- _raw: typeof row[2] === "string" ? JSON.parse(row[2]) : row[2],
894
+ _raw: rawParsed,
668
895
  timestamp: row[3] !== null ? jsToDate(row[3]) : null,
669
896
  level: row[4],
670
897
  message: row[5],
@@ -675,25 +902,6 @@ function rowToLogEntry(row) {
675
902
  source: row[10]
676
903
  };
677
904
  }
678
- const FIELD_MAPPINGS = {
679
- timestamp: ["timestamp", "ts", "time", "@timestamp", "datetime"],
680
- level: ["level", "severity", "loglevel", "lvl", "priority"],
681
- message: ["message", "msg", "body", "text"],
682
- service: ["service", "svc", "app", "component", "logger"],
683
- trace_id: ["trace_id", "traceId", "request_id", "correlation_id"]
684
- };
685
- const REVERSE_FIELD_MAP = (() => {
686
- const map = /* @__PURE__ */ new Map();
687
- for (const [canonical, aliases] of Object.entries(FIELD_MAPPINGS)) {
688
- for (const alias of aliases) {
689
- map.set(alias, canonical);
690
- }
691
- }
692
- return map;
693
- })();
694
- function resolveField(fieldName) {
695
- return REVERSE_FIELD_MAP.get(fieldName) ?? null;
696
- }
697
905
  class Ingester {
698
906
  db;
699
907
  options;
@@ -704,6 +912,11 @@ class Ingester {
704
912
  rl = null;
705
913
  stopped = false;
706
914
  flushing = null;
915
+ // Sliding window for ingestion rate calculation
916
+ static RATE_WINDOW_MS = 1e4;
917
+ // 10 seconds
918
+ batchRecords = [];
919
+ lastBatchTime = null;
707
920
  constructor(db, options) {
708
921
  this.db = db;
709
922
  this.options = options;
@@ -763,6 +976,23 @@ class Ingester {
763
976
  on(event, listener) {
764
977
  this.emitter.on(event, listener);
765
978
  }
979
+ /**
980
+ * Returns the current ingestion statistics including the smoothed
981
+ * ingestion rate (logs/second) over a 10-second sliding window.
982
+ */
983
+ getIngestionStats() {
984
+ const now = Date.now();
985
+ const windowStart = now - Ingester.RATE_WINDOW_MS;
986
+ this.batchRecords = this.batchRecords.filter(
987
+ (r) => r.timestamp >= windowStart
988
+ );
989
+ const totalLogs = this.batchRecords.reduce((sum, r) => sum + r.size, 0);
990
+ const ingestionRate = totalLogs / (Ingester.RATE_WINDOW_MS / 1e3);
991
+ return {
992
+ ingestionRate,
993
+ lastBatchTime: this.lastBatchTime
994
+ };
995
+ }
766
996
  // -------------------------------------------------------------------------
767
997
  // Line processing
768
998
  // -------------------------------------------------------------------------
@@ -770,15 +1000,18 @@ class Ingester {
770
1000
  const trimmed = line.trim();
771
1001
  if (trimmed === "") return;
772
1002
  let parsed;
1003
+ let raw;
773
1004
  try {
774
1005
  parsed = JSON.parse(trimmed);
1006
+ raw = trimmed;
775
1007
  } catch {
776
- return;
1008
+ parsed = { message: trimmed, level: "INFO", timestamp: (/* @__PURE__ */ new Date()).toISOString() };
1009
+ raw = JSON.stringify(parsed);
777
1010
  }
778
1011
  if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
779
1012
  return;
780
1013
  }
781
- const normalized = this.normalize(parsed, trimmed);
1014
+ const normalized = this.normalize(parsed, raw);
782
1015
  this.buffer.push(normalized);
783
1016
  if (this.buffer.length >= this.options.batchSize) {
784
1017
  this.flushBuffer();
@@ -843,6 +1076,9 @@ class Ingester {
843
1076
  try {
844
1077
  await this.db.insertBatch(batch);
845
1078
  await this.db.evictOldRows(this.options.maxRows);
1079
+ const now = Date.now();
1080
+ this.batchRecords.push({ timestamp: now, size: batch.length });
1081
+ this.lastBatchTime = new Date(now);
846
1082
  this.emitter.emit("batch", batch);
847
1083
  } catch (err) {
848
1084
  console.error("[ingester] doFlush error:", err);
@@ -864,13 +1100,16 @@ class Ingester {
864
1100
  }
865
1101
  }
866
1102
  function matchesFilter(log, filter) {
867
- if (filter.level !== void 0 && log.level !== filter.level) {
1103
+ if (filter.level && filter.level.length > 0 && !filter.level.map((l) => l.toUpperCase()).includes(log.level?.toUpperCase())) {
1104
+ return false;
1105
+ }
1106
+ if (filter.service && filter.service.length > 0 && !filter.service.includes(log.service ?? "")) {
868
1107
  return false;
869
1108
  }
870
- if (filter.service !== void 0 && log.service !== filter.service) {
1109
+ if (filter.host && filter.host.length > 0 && !filter.host.includes(log.host ?? "")) {
871
1110
  return false;
872
1111
  }
873
- if (filter.source !== void 0 && log.source !== filter.source) {
1112
+ if (filter.source && filter.source.length > 0 && !filter.source.includes(log.source)) {
874
1113
  return false;
875
1114
  }
876
1115
  if (filter.search !== void 0 && filter.search !== "") {
@@ -910,7 +1149,8 @@ function serializeStats(stats) {
910
1149
  timeRange: {
911
1150
  min: stats.timeRange.min?.toISOString() ?? null,
912
1151
  max: stats.timeRange.max?.toISOString() ?? null
913
- }
1152
+ },
1153
+ ingestionRate: stats.ingestionRate
914
1154
  };
915
1155
  }
916
1156
  class WSHandler {
@@ -940,6 +1180,8 @@ class WSHandler {
940
1180
  }
941
1181
  }
942
1182
  if (stats) {
1183
+ const ingestionStats = ingester.getIngestionStats();
1184
+ stats.ingestionRate = ingestionStats.ingestionRate;
943
1185
  this.broadcast([...logs], stats);
944
1186
  }
945
1187
  });
@@ -976,14 +1218,25 @@ class WSHandler {
976
1218
  }
977
1219
  const filterObj = msg.filter;
978
1220
  const filter = {};
979
- if (typeof filterObj.level === "string") {
980
- filter.level = filterObj.level;
1221
+ if (Array.isArray(filterObj.level)) {
1222
+ filter.level = filterObj.level.filter((v) => typeof v === "string");
1223
+ } else if (typeof filterObj.level === "string") {
1224
+ filter.level = [filterObj.level];
981
1225
  }
982
- if (typeof filterObj.service === "string") {
983
- filter.service = filterObj.service;
1226
+ if (Array.isArray(filterObj.service)) {
1227
+ filter.service = filterObj.service.filter((v) => typeof v === "string");
1228
+ } else if (typeof filterObj.service === "string") {
1229
+ filter.service = [filterObj.service];
984
1230
  }
985
- if (typeof filterObj.source === "string") {
986
- filter.source = filterObj.source;
1231
+ if (Array.isArray(filterObj.host)) {
1232
+ filter.host = filterObj.host.filter((v) => typeof v === "string");
1233
+ } else if (typeof filterObj.host === "string") {
1234
+ filter.host = [filterObj.host];
1235
+ }
1236
+ if (Array.isArray(filterObj.source)) {
1237
+ filter.source = filterObj.source.filter((v) => typeof v === "string");
1238
+ } else if (typeof filterObj.source === "string") {
1239
+ filter.source = [filterObj.source];
987
1240
  }
988
1241
  if (typeof filterObj.search === "string") {
989
1242
  filter.search = filterObj.search;
@@ -1037,6 +1290,7 @@ function parseRelayArgs(args) {
1037
1290
  service: { type: "string", short: "s" },
1038
1291
  "batch-size": { type: "string" },
1039
1292
  interval: { type: "string" },
1293
+ "max-retries": { type: "string" },
1040
1294
  help: { type: "boolean", short: "h" }
1041
1295
  },
1042
1296
  strict: true
@@ -1046,6 +1300,7 @@ function parseRelayArgs(args) {
1046
1300
  service: values.service,
1047
1301
  batchSize: values["batch-size"] !== void 0 ? parseInt(values["batch-size"], 10) : 100,
1048
1302
  intervalMs: values.interval !== void 0 ? parseInt(values.interval, 10) : 500,
1303
+ maxRetries: values["max-retries"] !== void 0 ? parseInt(values["max-retries"], 10) : 3,
1049
1304
  help: values.help ?? false
1050
1305
  };
1051
1306
  }
@@ -1059,6 +1314,7 @@ Options:
1059
1314
  -s, --service <name> Service name to inject into each log entry
1060
1315
  --batch-size <n> Lines per HTTP batch (default: 100)
1061
1316
  --interval <ms> Flush interval in milliseconds (default: 500)
1317
+ --max-retries <n> Max retry attempts on transient failure (default: 3)
1062
1318
  -h, --help Show this help message
1063
1319
 
1064
1320
  Examples:
@@ -1066,6 +1322,75 @@ Examples:
1066
1322
  cat app.log | lduck relay --url http://lduck:9090 --service backend
1067
1323
  docker logs -f myapp | lduck relay -s myapp -u http://localhost:8080
1068
1324
  `.trimStart();
1325
+ function parseLine(line, service) {
1326
+ const trimmed = line.trim();
1327
+ if (trimmed === "") return null;
1328
+ let parsed;
1329
+ try {
1330
+ parsed = JSON.parse(trimmed);
1331
+ } catch {
1332
+ parsed = { message: trimmed, level: "INFO", timestamp: (/* @__PURE__ */ new Date()).toISOString() };
1333
+ }
1334
+ if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
1335
+ return null;
1336
+ }
1337
+ if (service !== void 0 && !("service" in parsed)) {
1338
+ parsed.service = service;
1339
+ }
1340
+ return parsed;
1341
+ }
1342
+ function computeBackoffDelay(attempt, baseDelay, maxDelay) {
1343
+ return Math.min(baseDelay * 2 ** attempt, maxDelay);
1344
+ }
1345
+ function accumulateFlushResult(summary, result) {
1346
+ return {
1347
+ totalSent: summary.totalSent + result.accepted,
1348
+ totalErrors: summary.totalErrors + result.errors,
1349
+ totalRetries: summary.totalRetries + result.retries
1350
+ };
1351
+ }
1352
+ function formatRelaySummary(summary) {
1353
+ return `[relay] Done. Sent: ${summary.totalSent}, Errors: ${summary.totalErrors}, Retries: ${summary.totalRetries}`;
1354
+ }
1355
+ const DEFAULT_BASE_DELAY = 100;
1356
+ const DEFAULT_MAX_DELAY = 3e4;
1357
+ function isRetryableStatus(status) {
1358
+ return status >= 500;
1359
+ }
1360
+ async function flushWithRetry(batch, ingestUrl, maxRetries, fetchFn = globalThis.fetch, backoffOptions) {
1361
+ const baseDelay = DEFAULT_BASE_DELAY;
1362
+ const maxDelay = DEFAULT_MAX_DELAY;
1363
+ let retries = 0;
1364
+ for (let attempt = 0; attempt <= maxRetries; attempt++) {
1365
+ if (attempt > 0) {
1366
+ const delay = computeBackoffDelay(attempt - 1, baseDelay, maxDelay);
1367
+ await new Promise((resolve) => setTimeout(resolve, delay));
1368
+ retries++;
1369
+ }
1370
+ try {
1371
+ const res = await fetchFn(ingestUrl, {
1372
+ method: "POST",
1373
+ headers: { "Content-Type": "application/json" },
1374
+ body: JSON.stringify(batch)
1375
+ });
1376
+ if (res.ok) {
1377
+ const result = await res.json();
1378
+ return { accepted: result.accepted, errors: 0, retries };
1379
+ }
1380
+ if (!isRetryableStatus(res.status)) {
1381
+ const text2 = await res.text();
1382
+ console.error(`[relay] HTTP ${res.status}: ${text2}`);
1383
+ return { accepted: 0, errors: batch.length, retries: 0 };
1384
+ }
1385
+ const text = await res.text();
1386
+ console.error(`[relay] HTTP ${res.status}: ${text} (attempt ${attempt + 1}/${maxRetries + 1})`);
1387
+ } catch (err) {
1388
+ const message = err instanceof Error ? err.message : String(err);
1389
+ console.error(`[relay] Send failed: ${message} (attempt ${attempt + 1}/${maxRetries + 1})`);
1390
+ }
1391
+ }
1392
+ return { accepted: 0, errors: batch.length, retries };
1393
+ }
1069
1394
  async function runRelay(args) {
1070
1395
  const opts = parseRelayArgs(args);
1071
1396
  if (opts.help) {
@@ -1075,32 +1400,14 @@ async function runRelay(args) {
1075
1400
  const ingestUrl = opts.url.replace(/\/$/, "") + "/api/ingest";
1076
1401
  let buffer = [];
1077
1402
  let flushTimer = null;
1078
- let totalSent = 0;
1079
- let totalErrors = 0;
1403
+ let summary = { totalSent: 0, totalErrors: 0, totalRetries: 0 };
1080
1404
  let stdinClosed = false;
1081
1405
  async function flush() {
1082
1406
  if (buffer.length === 0) return;
1083
1407
  const batch = buffer;
1084
1408
  buffer = [];
1085
- try {
1086
- const res = await fetch(ingestUrl, {
1087
- method: "POST",
1088
- headers: { "Content-Type": "application/json" },
1089
- body: JSON.stringify(batch)
1090
- });
1091
- if (!res.ok) {
1092
- const text = await res.text();
1093
- console.error(`[relay] HTTP ${res.status}: ${text}`);
1094
- totalErrors += batch.length;
1095
- } else {
1096
- const result = await res.json();
1097
- totalSent += result.accepted;
1098
- }
1099
- } catch (err) {
1100
- const message = err instanceof Error ? err.message : String(err);
1101
- console.error(`[relay] Send failed: ${message}`);
1102
- totalErrors += batch.length;
1103
- }
1409
+ const result = await flushWithRetry(batch, ingestUrl, opts.maxRetries);
1410
+ summary = accumulateFlushResult(summary, result);
1104
1411
  }
1105
1412
  function resetFlushTimer() {
1106
1413
  if (flushTimer !== null) {
@@ -1121,20 +1428,8 @@ async function runRelay(args) {
1121
1428
  console.error(`[relay] Relaying to ${ingestUrl}${opts.service ? ` (service: ${opts.service})` : ""}`);
1122
1429
  resetFlushTimer();
1123
1430
  rl.on("line", (line) => {
1124
- const trimmed = line.trim();
1125
- if (trimmed === "") return;
1126
- let parsed;
1127
- try {
1128
- parsed = JSON.parse(trimmed);
1129
- } catch {
1130
- return;
1131
- }
1132
- if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
1133
- return;
1134
- }
1135
- if (opts.service !== void 0 && !("service" in parsed)) {
1136
- parsed.service = opts.service;
1137
- }
1431
+ const parsed = parseLine(line, opts.service);
1432
+ if (parsed === null) return;
1138
1433
  buffer.push(parsed);
1139
1434
  if (buffer.length >= opts.batchSize) {
1140
1435
  flush();
@@ -1148,11 +1443,7 @@ async function runRelay(args) {
1148
1443
  flushTimer = null;
1149
1444
  }
1150
1445
  await flush();
1151
- if (totalSent > 0 || totalErrors > 0) {
1152
- console.error(
1153
- `[relay] Done. Sent: ${totalSent}, Errors: ${totalErrors}`
1154
- );
1155
- }
1446
+ console.error(formatRelaySummary(summary));
1156
1447
  resolve();
1157
1448
  });
1158
1449
  });