@vaultgradient/pipequery-cli 0.3.3 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +151 -1
- package/__tests__/nl-translator.test.ts +228 -0
- package/__tests__/telegram-format.test.ts +62 -0
- package/__tests__/watch-manager.test.ts +202 -0
- package/dist/index.js +1938 -80
- package/package.json +12 -2
- package/src/commands/mcp.ts +87 -0
- package/src/commands/source.ts +63 -1
- package/src/commands/telegram.ts +65 -0
- package/src/commands/watch.ts +84 -0
- package/src/config/schema.ts +153 -1
- package/src/index.ts +80 -6
- package/src/mcp/http.ts +191 -0
- package/src/mcp/provider.ts +223 -0
- package/src/mcp/server.ts +138 -0
- package/src/mcp/stdio.ts +26 -0
- package/src/server/app.ts +10 -1
- package/src/server/sources/kafka.ts +166 -0
- package/src/server/sources/manager.ts +12 -0
- package/src/server/sources/mysql.ts +128 -0
- package/src/server/sources/postgres.ts +139 -0
- package/src/server/sources/sqlite.ts +114 -0
- package/src/server/watches/manager.ts +177 -0
- package/src/telegram/bot.ts +247 -0
- package/src/telegram/format.ts +158 -0
- package/src/telegram/nl.ts +318 -0
- package/src/telegram/notifier.ts +110 -0
package/dist/index.js
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
// src/index.ts
|
|
4
4
|
import { Command } from "commander";
|
|
5
|
-
import
|
|
5
|
+
import chalk9 from "chalk";
|
|
6
6
|
|
|
7
7
|
// src/commands/init.ts
|
|
8
8
|
import { writeFile } from "fs/promises";
|
|
@@ -83,7 +83,7 @@ async function initCommand() {
|
|
|
83
83
|
|
|
84
84
|
// src/commands/serve.ts
|
|
85
85
|
import { writeFile as writeFile3, mkdir, readFile as readFile3, unlink } from "fs/promises";
|
|
86
|
-
import { resolve as
|
|
86
|
+
import { resolve as resolve5 } from "path";
|
|
87
87
|
import { existsSync as existsSync3 } from "fs";
|
|
88
88
|
import { spawn } from "child_process";
|
|
89
89
|
|
|
@@ -236,11 +236,11 @@ var WebSocketSourceAdapter = class {
|
|
|
236
236
|
listeners = /* @__PURE__ */ new Set();
|
|
237
237
|
maxBuffer;
|
|
238
238
|
async start() {
|
|
239
|
-
return new Promise((
|
|
239
|
+
return new Promise((resolve10, reject) => {
|
|
240
240
|
this.ws = new WebSocket(this.config.url);
|
|
241
241
|
this.ws.on("open", () => {
|
|
242
242
|
this.error = void 0;
|
|
243
|
-
|
|
243
|
+
resolve10();
|
|
244
244
|
});
|
|
245
245
|
this.ws.on("message", (raw) => {
|
|
246
246
|
try {
|
|
@@ -394,6 +394,455 @@ var StaticSourceAdapter = class {
|
|
|
394
394
|
}
|
|
395
395
|
};
|
|
396
396
|
|
|
397
|
+
// src/server/sources/postgres.ts
|
|
398
|
+
import pg from "pg";
|
|
399
|
+
var DEFAULT_INTERVAL_MS = 3e4;
|
|
400
|
+
var DEFAULT_MAX_ROWS = 1e4;
|
|
401
|
+
function expandEnv(input) {
|
|
402
|
+
return input.replace(/\$\{([A-Z0-9_]+)\}/gi, (_, name) => {
|
|
403
|
+
const value = process.env[name];
|
|
404
|
+
if (value === void 0) {
|
|
405
|
+
process.stderr.write(
|
|
406
|
+
`[pipequery] env var "${name}" referenced in postgres source URL is not set
|
|
407
|
+
`
|
|
408
|
+
);
|
|
409
|
+
return "";
|
|
410
|
+
}
|
|
411
|
+
return value;
|
|
412
|
+
});
|
|
413
|
+
}
|
|
414
|
+
function resolveSsl(ssl) {
|
|
415
|
+
if (ssl === false) return false;
|
|
416
|
+
if (ssl === "no-verify") return { rejectUnauthorized: false };
|
|
417
|
+
return { rejectUnauthorized: true };
|
|
418
|
+
}
|
|
419
|
+
var PostgresSourceAdapter = class {
|
|
420
|
+
constructor(config) {
|
|
421
|
+
this.config = config;
|
|
422
|
+
this.intervalMs = config.interval ? parseDuration(config.interval) : DEFAULT_INTERVAL_MS;
|
|
423
|
+
this.maxRows = config.maxRows ?? DEFAULT_MAX_ROWS;
|
|
424
|
+
}
|
|
425
|
+
config;
|
|
426
|
+
pool = null;
|
|
427
|
+
data = [];
|
|
428
|
+
timer = null;
|
|
429
|
+
lastFetch = null;
|
|
430
|
+
error;
|
|
431
|
+
listeners = /* @__PURE__ */ new Set();
|
|
432
|
+
intervalMs;
|
|
433
|
+
maxRows;
|
|
434
|
+
async start() {
|
|
435
|
+
const connectionString = expandEnv(this.config.url);
|
|
436
|
+
this.pool = new pg.Pool({
|
|
437
|
+
connectionString,
|
|
438
|
+
ssl: resolveSsl(this.config.ssl),
|
|
439
|
+
// Small pool — we're read-only polling, not a transactional app.
|
|
440
|
+
min: 0,
|
|
441
|
+
max: 2,
|
|
442
|
+
idleTimeoutMillis: 1e4
|
|
443
|
+
});
|
|
444
|
+
this.pool.on("error", (err) => {
|
|
445
|
+
this.error = err.message;
|
|
446
|
+
});
|
|
447
|
+
await this.fetch();
|
|
448
|
+
this.timer = setInterval(() => {
|
|
449
|
+
void this.fetch();
|
|
450
|
+
}, this.intervalMs);
|
|
451
|
+
}
|
|
452
|
+
stop() {
|
|
453
|
+
if (this.timer) {
|
|
454
|
+
clearInterval(this.timer);
|
|
455
|
+
this.timer = null;
|
|
456
|
+
}
|
|
457
|
+
if (this.pool) {
|
|
458
|
+
void this.pool.end();
|
|
459
|
+
this.pool = null;
|
|
460
|
+
}
|
|
461
|
+
}
|
|
462
|
+
getData() {
|
|
463
|
+
return this.data;
|
|
464
|
+
}
|
|
465
|
+
getStatus() {
|
|
466
|
+
return {
|
|
467
|
+
healthy: !this.error,
|
|
468
|
+
rowCount: this.data.length,
|
|
469
|
+
lastFetch: this.lastFetch,
|
|
470
|
+
error: this.error
|
|
471
|
+
};
|
|
472
|
+
}
|
|
473
|
+
onUpdate(callback) {
|
|
474
|
+
this.listeners.add(callback);
|
|
475
|
+
return () => this.listeners.delete(callback);
|
|
476
|
+
}
|
|
477
|
+
async refresh() {
|
|
478
|
+
await this.fetch();
|
|
479
|
+
}
|
|
480
|
+
async fetch() {
|
|
481
|
+
if (!this.pool) {
|
|
482
|
+
this.error = "Postgres pool not initialized";
|
|
483
|
+
return;
|
|
484
|
+
}
|
|
485
|
+
try {
|
|
486
|
+
const capQuery = `SELECT * FROM (${stripTrailingSemicolon(this.config.query)}) AS pq_src LIMIT ${this.maxRows + 1}`;
|
|
487
|
+
const res = await this.pool.query(capQuery);
|
|
488
|
+
if (res.rows.length > this.maxRows) {
|
|
489
|
+
throw new Error(
|
|
490
|
+
`Query returned more than maxRows=${this.maxRows}. Add LIMIT/WHERE to bound the result set, or raise maxRows in the source config.`
|
|
491
|
+
);
|
|
492
|
+
}
|
|
493
|
+
this.data = res.rows;
|
|
494
|
+
this.lastFetch = /* @__PURE__ */ new Date();
|
|
495
|
+
this.error = void 0;
|
|
496
|
+
this.notify();
|
|
497
|
+
} catch (err) {
|
|
498
|
+
this.error = err instanceof Error ? err.message : String(err);
|
|
499
|
+
}
|
|
500
|
+
}
|
|
501
|
+
notify() {
|
|
502
|
+
for (const cb of this.listeners) cb(this.data);
|
|
503
|
+
}
|
|
504
|
+
};
|
|
505
|
+
function stripTrailingSemicolon(sql) {
|
|
506
|
+
return sql.trim().replace(/;+\s*$/, "");
|
|
507
|
+
}
|
|
508
|
+
|
|
509
|
+
// src/server/sources/mysql.ts
|
|
510
|
+
import mysql from "mysql2/promise";
|
|
511
|
+
var DEFAULT_INTERVAL_MS2 = 3e4;
|
|
512
|
+
var DEFAULT_MAX_ROWS2 = 1e4;
|
|
513
|
+
function expandEnv2(input) {
|
|
514
|
+
return input.replace(/\$\{([A-Z0-9_]+)\}/gi, (_, name) => {
|
|
515
|
+
const value = process.env[name];
|
|
516
|
+
if (value === void 0) {
|
|
517
|
+
process.stderr.write(
|
|
518
|
+
`[pipequery] env var "${name}" referenced in mysql source URL is not set
|
|
519
|
+
`
|
|
520
|
+
);
|
|
521
|
+
return "";
|
|
522
|
+
}
|
|
523
|
+
return value;
|
|
524
|
+
});
|
|
525
|
+
}
|
|
526
|
+
function resolveSsl2(ssl) {
|
|
527
|
+
if (ssl === false) return void 0;
|
|
528
|
+
if (ssl === "no-verify") return { rejectUnauthorized: false };
|
|
529
|
+
return { rejectUnauthorized: true };
|
|
530
|
+
}
|
|
531
|
+
var MysqlSourceAdapter = class {
|
|
532
|
+
constructor(config) {
|
|
533
|
+
this.config = config;
|
|
534
|
+
this.intervalMs = config.interval ? parseDuration(config.interval) : DEFAULT_INTERVAL_MS2;
|
|
535
|
+
this.maxRows = config.maxRows ?? DEFAULT_MAX_ROWS2;
|
|
536
|
+
}
|
|
537
|
+
config;
|
|
538
|
+
pool = null;
|
|
539
|
+
data = [];
|
|
540
|
+
timer = null;
|
|
541
|
+
lastFetch = null;
|
|
542
|
+
error;
|
|
543
|
+
listeners = /* @__PURE__ */ new Set();
|
|
544
|
+
intervalMs;
|
|
545
|
+
maxRows;
|
|
546
|
+
async start() {
|
|
547
|
+
const uri = expandEnv2(this.config.url);
|
|
548
|
+
this.pool = mysql.createPool({
|
|
549
|
+
uri,
|
|
550
|
+
ssl: resolveSsl2(this.config.ssl),
|
|
551
|
+
connectionLimit: 2,
|
|
552
|
+
waitForConnections: true,
|
|
553
|
+
// mysql2 returns BigInt for large integers by default — stringify so the
|
|
554
|
+
// data serializes cleanly to JSON (which can't encode BigInt).
|
|
555
|
+
supportBigNumbers: true,
|
|
556
|
+
bigNumberStrings: true
|
|
557
|
+
});
|
|
558
|
+
await this.fetch();
|
|
559
|
+
this.timer = setInterval(() => {
|
|
560
|
+
void this.fetch();
|
|
561
|
+
}, this.intervalMs);
|
|
562
|
+
}
|
|
563
|
+
stop() {
|
|
564
|
+
if (this.timer) {
|
|
565
|
+
clearInterval(this.timer);
|
|
566
|
+
this.timer = null;
|
|
567
|
+
}
|
|
568
|
+
if (this.pool) {
|
|
569
|
+
void this.pool.end();
|
|
570
|
+
this.pool = null;
|
|
571
|
+
}
|
|
572
|
+
}
|
|
573
|
+
getData() {
|
|
574
|
+
return this.data;
|
|
575
|
+
}
|
|
576
|
+
getStatus() {
|
|
577
|
+
return {
|
|
578
|
+
healthy: !this.error,
|
|
579
|
+
rowCount: this.data.length,
|
|
580
|
+
lastFetch: this.lastFetch,
|
|
581
|
+
error: this.error
|
|
582
|
+
};
|
|
583
|
+
}
|
|
584
|
+
onUpdate(callback) {
|
|
585
|
+
this.listeners.add(callback);
|
|
586
|
+
return () => this.listeners.delete(callback);
|
|
587
|
+
}
|
|
588
|
+
async refresh() {
|
|
589
|
+
await this.fetch();
|
|
590
|
+
}
|
|
591
|
+
async fetch() {
|
|
592
|
+
if (!this.pool) {
|
|
593
|
+
this.error = "MySQL pool not initialized";
|
|
594
|
+
return;
|
|
595
|
+
}
|
|
596
|
+
try {
|
|
597
|
+
const capQuery = `SELECT * FROM (${stripTrailingSemicolon2(this.config.query)}) AS pq_src LIMIT ${this.maxRows + 1}`;
|
|
598
|
+
const [rows] = await this.pool.query(capQuery);
|
|
599
|
+
const asArray = Array.isArray(rows) ? rows : [];
|
|
600
|
+
if (asArray.length > this.maxRows) {
|
|
601
|
+
throw new Error(
|
|
602
|
+
`Query returned more than maxRows=${this.maxRows}. Add LIMIT/WHERE to bound the result set, or raise maxRows in the source config.`
|
|
603
|
+
);
|
|
604
|
+
}
|
|
605
|
+
this.data = asArray;
|
|
606
|
+
this.lastFetch = /* @__PURE__ */ new Date();
|
|
607
|
+
this.error = void 0;
|
|
608
|
+
this.notify();
|
|
609
|
+
} catch (err) {
|
|
610
|
+
this.error = err instanceof Error ? err.message : String(err);
|
|
611
|
+
}
|
|
612
|
+
}
|
|
613
|
+
notify() {
|
|
614
|
+
for (const cb of this.listeners) cb(this.data);
|
|
615
|
+
}
|
|
616
|
+
};
|
|
617
|
+
function stripTrailingSemicolon2(sql) {
|
|
618
|
+
return sql.trim().replace(/;+\s*$/, "");
|
|
619
|
+
}
|
|
620
|
+
|
|
621
|
+
// src/server/sources/sqlite.ts
|
|
622
|
+
import { resolve as resolve4 } from "path";
|
|
623
|
+
import Database from "better-sqlite3";
|
|
624
|
+
var DEFAULT_INTERVAL_MS3 = 3e4;
|
|
625
|
+
var DEFAULT_MAX_ROWS3 = 1e4;
|
|
626
|
+
var IN_MEMORY = ":memory:";
|
|
627
|
+
var SqliteSourceAdapter = class {
|
|
628
|
+
constructor(config, cwd) {
|
|
629
|
+
this.config = config;
|
|
630
|
+
this.intervalMs = config.interval ? parseDuration(config.interval) : DEFAULT_INTERVAL_MS3;
|
|
631
|
+
this.maxRows = config.maxRows ?? DEFAULT_MAX_ROWS3;
|
|
632
|
+
this.readonly = config.readonly ?? true;
|
|
633
|
+
this.resolvedPath = config.path === IN_MEMORY ? IN_MEMORY : resolve4(cwd, config.path);
|
|
634
|
+
}
|
|
635
|
+
config;
|
|
636
|
+
db = null;
|
|
637
|
+
data = [];
|
|
638
|
+
timer = null;
|
|
639
|
+
lastFetch = null;
|
|
640
|
+
error;
|
|
641
|
+
listeners = /* @__PURE__ */ new Set();
|
|
642
|
+
intervalMs;
|
|
643
|
+
maxRows;
|
|
644
|
+
resolvedPath;
|
|
645
|
+
readonly;
|
|
646
|
+
async start() {
|
|
647
|
+
const opts = this.resolvedPath === IN_MEMORY ? {} : { readonly: this.readonly, fileMustExist: this.readonly };
|
|
648
|
+
this.db = new Database(this.resolvedPath, opts);
|
|
649
|
+
if (this.resolvedPath !== IN_MEMORY && !this.readonly) {
|
|
650
|
+
this.db.pragma("journal_mode = WAL");
|
|
651
|
+
}
|
|
652
|
+
await this.fetch();
|
|
653
|
+
this.timer = setInterval(() => {
|
|
654
|
+
void this.fetch();
|
|
655
|
+
}, this.intervalMs);
|
|
656
|
+
}
|
|
657
|
+
stop() {
|
|
658
|
+
if (this.timer) {
|
|
659
|
+
clearInterval(this.timer);
|
|
660
|
+
this.timer = null;
|
|
661
|
+
}
|
|
662
|
+
if (this.db) {
|
|
663
|
+
this.db.close();
|
|
664
|
+
this.db = null;
|
|
665
|
+
}
|
|
666
|
+
}
|
|
667
|
+
getData() {
|
|
668
|
+
return this.data;
|
|
669
|
+
}
|
|
670
|
+
getStatus() {
|
|
671
|
+
return {
|
|
672
|
+
healthy: !this.error,
|
|
673
|
+
rowCount: this.data.length,
|
|
674
|
+
lastFetch: this.lastFetch,
|
|
675
|
+
error: this.error
|
|
676
|
+
};
|
|
677
|
+
}
|
|
678
|
+
onUpdate(callback) {
|
|
679
|
+
this.listeners.add(callback);
|
|
680
|
+
return () => this.listeners.delete(callback);
|
|
681
|
+
}
|
|
682
|
+
async refresh() {
|
|
683
|
+
await this.fetch();
|
|
684
|
+
}
|
|
685
|
+
async fetch() {
|
|
686
|
+
if (!this.db) {
|
|
687
|
+
this.error = "SQLite database not initialized";
|
|
688
|
+
return;
|
|
689
|
+
}
|
|
690
|
+
try {
|
|
691
|
+
const capQuery = `SELECT * FROM (${stripTrailingSemicolon3(this.config.query)}) AS pq_src LIMIT ${this.maxRows + 1}`;
|
|
692
|
+
const rows = this.db.prepare(capQuery).all();
|
|
693
|
+
if (rows.length > this.maxRows) {
|
|
694
|
+
throw new Error(
|
|
695
|
+
`Query returned more than maxRows=${this.maxRows}. Add LIMIT/WHERE to bound the result set, or raise maxRows in the source config.`
|
|
696
|
+
);
|
|
697
|
+
}
|
|
698
|
+
this.data = rows;
|
|
699
|
+
this.lastFetch = /* @__PURE__ */ new Date();
|
|
700
|
+
this.error = void 0;
|
|
701
|
+
this.notify();
|
|
702
|
+
} catch (err) {
|
|
703
|
+
this.error = err instanceof Error ? err.message : String(err);
|
|
704
|
+
}
|
|
705
|
+
}
|
|
706
|
+
notify() {
|
|
707
|
+
for (const cb of this.listeners) cb(this.data);
|
|
708
|
+
}
|
|
709
|
+
};
|
|
710
|
+
function stripTrailingSemicolon3(sql) {
|
|
711
|
+
return sql.trim().replace(/;+\s*$/, "");
|
|
712
|
+
}
|
|
713
|
+
|
|
714
|
+
// src/server/sources/kafka.ts
|
|
715
|
+
import { randomBytes } from "crypto";
|
|
716
|
+
import { Kafka, logLevel } from "kafkajs";
|
|
717
|
+
var DEFAULT_MAX_BUFFER = 1e3;
|
|
718
|
+
function expandEnv3(input) {
|
|
719
|
+
return input.replace(/\$\{([A-Z0-9_]+)\}/gi, (_, name) => {
|
|
720
|
+
const value = process.env[name];
|
|
721
|
+
if (value === void 0) {
|
|
722
|
+
process.stderr.write(
|
|
723
|
+
`[pipequery] env var "${name}" referenced in kafka source is not set
|
|
724
|
+
`
|
|
725
|
+
);
|
|
726
|
+
return "";
|
|
727
|
+
}
|
|
728
|
+
return value;
|
|
729
|
+
});
|
|
730
|
+
}
|
|
731
|
+
function normalizeBrokers(brokers) {
|
|
732
|
+
const list = Array.isArray(brokers) ? brokers : brokers.split(",");
|
|
733
|
+
return list.map((b) => expandEnv3(b.trim())).filter(Boolean);
|
|
734
|
+
}
|
|
735
|
+
var KafkaSourceAdapter = class {
|
|
736
|
+
constructor(config) {
|
|
737
|
+
this.config = config;
|
|
738
|
+
this.maxBuffer = config.maxBuffer ?? DEFAULT_MAX_BUFFER;
|
|
739
|
+
this.groupId = config.groupId ?? `pipequery-${randomBytes(6).toString("hex")}`;
|
|
740
|
+
}
|
|
741
|
+
config;
|
|
742
|
+
data = [];
|
|
743
|
+
kafka = null;
|
|
744
|
+
consumer = null;
|
|
745
|
+
lastFetch = null;
|
|
746
|
+
error;
|
|
747
|
+
connected = false;
|
|
748
|
+
listeners = /* @__PURE__ */ new Set();
|
|
749
|
+
maxBuffer;
|
|
750
|
+
groupId;
|
|
751
|
+
async start() {
|
|
752
|
+
const brokers = normalizeBrokers(this.config.brokers);
|
|
753
|
+
if (brokers.length === 0) {
|
|
754
|
+
throw new Error("Kafka source has no brokers configured");
|
|
755
|
+
}
|
|
756
|
+
this.kafka = new Kafka({
|
|
757
|
+
clientId: `pipequery-${this.config.topic}`,
|
|
758
|
+
brokers,
|
|
759
|
+
ssl: this.config.ssl ?? false,
|
|
760
|
+
sasl: this.config.sasl ? {
|
|
761
|
+
mechanism: this.config.sasl.mechanism,
|
|
762
|
+
username: expandEnv3(this.config.sasl.username),
|
|
763
|
+
password: expandEnv3(this.config.sasl.password)
|
|
764
|
+
} : void 0,
|
|
765
|
+
// Silence kafkajs's own logger — we surface errors via SourceStatus.
|
|
766
|
+
logLevel: logLevel.ERROR
|
|
767
|
+
});
|
|
768
|
+
this.consumer = this.kafka.consumer({ groupId: this.groupId });
|
|
769
|
+
await this.consumer.connect();
|
|
770
|
+
await this.consumer.subscribe({
|
|
771
|
+
topic: this.config.topic,
|
|
772
|
+
fromBeginning: this.config.fromBeginning ?? false
|
|
773
|
+
});
|
|
774
|
+
this.connected = true;
|
|
775
|
+
this.error = void 0;
|
|
776
|
+
void this.consumer.run({
|
|
777
|
+
eachMessage: async ({ message, partition }) => {
|
|
778
|
+
try {
|
|
779
|
+
const row = this.decode(message.value, message.key, message.timestamp, partition, message.offset);
|
|
780
|
+
this.data.push(row);
|
|
781
|
+
if (this.data.length > this.maxBuffer) {
|
|
782
|
+
this.data = this.data.slice(-this.maxBuffer);
|
|
783
|
+
}
|
|
784
|
+
this.lastFetch = /* @__PURE__ */ new Date();
|
|
785
|
+
this.error = void 0;
|
|
786
|
+
this.notify();
|
|
787
|
+
} catch {
|
|
788
|
+
}
|
|
789
|
+
}
|
|
790
|
+
});
|
|
791
|
+
}
|
|
792
|
+
stop() {
|
|
793
|
+
this.connected = false;
|
|
794
|
+
if (this.consumer) {
|
|
795
|
+
const consumer = this.consumer;
|
|
796
|
+
this.consumer = null;
|
|
797
|
+
void consumer.disconnect().catch(() => {
|
|
798
|
+
});
|
|
799
|
+
}
|
|
800
|
+
this.kafka = null;
|
|
801
|
+
}
|
|
802
|
+
getData() {
|
|
803
|
+
return this.data;
|
|
804
|
+
}
|
|
805
|
+
getStatus() {
|
|
806
|
+
return {
|
|
807
|
+
healthy: this.connected && !this.error,
|
|
808
|
+
rowCount: this.data.length,
|
|
809
|
+
lastFetch: this.lastFetch,
|
|
810
|
+
error: this.error
|
|
811
|
+
};
|
|
812
|
+
}
|
|
813
|
+
onUpdate(callback) {
|
|
814
|
+
this.listeners.add(callback);
|
|
815
|
+
return () => this.listeners.delete(callback);
|
|
816
|
+
}
|
|
817
|
+
decode(value, key, timestamp, partition, offset) {
|
|
818
|
+
const base = {
|
|
819
|
+
_kafka_topic: this.config.topic,
|
|
820
|
+
_kafka_partition: partition,
|
|
821
|
+
_kafka_offset: offset,
|
|
822
|
+
_kafka_timestamp: Number.parseInt(timestamp, 10),
|
|
823
|
+
_kafka_key: key ? key.toString("utf8") : null
|
|
824
|
+
};
|
|
825
|
+
if (!value) {
|
|
826
|
+
return { ...base, value: null };
|
|
827
|
+
}
|
|
828
|
+
const format = this.config.valueFormat ?? "json";
|
|
829
|
+
if (format === "string") {
|
|
830
|
+
return { ...base, value: value.toString("utf8") };
|
|
831
|
+
}
|
|
832
|
+
if (format === "raw") {
|
|
833
|
+
return { ...base, value: value.toString("base64") };
|
|
834
|
+
}
|
|
835
|
+
const decoded = JSON.parse(value.toString("utf8"));
|
|
836
|
+
if (decoded !== null && typeof decoded === "object" && !Array.isArray(decoded)) {
|
|
837
|
+
return { ...base, ...decoded };
|
|
838
|
+
}
|
|
839
|
+
return { ...base, value: decoded };
|
|
840
|
+
}
|
|
841
|
+
notify() {
|
|
842
|
+
for (const cb of this.listeners) cb(this.data);
|
|
843
|
+
}
|
|
844
|
+
};
|
|
845
|
+
|
|
397
846
|
// src/server/sources/manager.ts
|
|
398
847
|
var SourceManager = class {
|
|
399
848
|
constructor(cwd) {
|
|
@@ -479,25 +928,22 @@ var SourceManager = class {
|
|
|
479
928
|
return new FileSourceAdapter(config, this.cwd);
|
|
480
929
|
case "static":
|
|
481
930
|
return new StaticSourceAdapter(config);
|
|
931
|
+
case "postgres":
|
|
932
|
+
return new PostgresSourceAdapter(config);
|
|
933
|
+
case "mysql":
|
|
934
|
+
return new MysqlSourceAdapter(config);
|
|
935
|
+
case "sqlite":
|
|
936
|
+
return new SqliteSourceAdapter(config, this.cwd);
|
|
937
|
+
case "kafka":
|
|
938
|
+
return new KafkaSourceAdapter(config);
|
|
482
939
|
default:
|
|
483
940
|
throw new Error(`Unknown source type: ${config.type}`);
|
|
484
941
|
}
|
|
485
942
|
}
|
|
486
943
|
};
|
|
487
944
|
|
|
488
|
-
// src/server/
|
|
489
|
-
|
|
490
|
-
app.get("/health", async () => {
|
|
491
|
-
return { status: "ok", uptime: Date.now() - startedAt.getTime() };
|
|
492
|
-
});
|
|
493
|
-
app.get("/status", async () => {
|
|
494
|
-
return {
|
|
495
|
-
status: "ok",
|
|
496
|
-
uptime: Date.now() - startedAt.getTime(),
|
|
497
|
-
sources: sourceManager.getAllStatuses()
|
|
498
|
-
};
|
|
499
|
-
});
|
|
500
|
-
}
|
|
945
|
+
// src/server/watches/manager.ts
|
|
946
|
+
import { createHash } from "crypto";
|
|
501
947
|
|
|
502
948
|
// ../src/engine/types.ts
|
|
503
949
|
var AGGREGATE_NAMES = /* @__PURE__ */ new Set([
|
|
@@ -2325,6 +2771,333 @@ function query(context, expression) {
|
|
|
2325
2771
|
return fn(context);
|
|
2326
2772
|
}
|
|
2327
2773
|
|
|
2774
|
+
// src/server/activity-log.ts
|
|
2775
|
+
var MAX_ENTRIES = 100;
|
|
2776
|
+
var entries = [];
|
|
2777
|
+
function logActivity(level, message) {
|
|
2778
|
+
entries.push({
|
|
2779
|
+
time: (/* @__PURE__ */ new Date()).toISOString(),
|
|
2780
|
+
level,
|
|
2781
|
+
message
|
|
2782
|
+
});
|
|
2783
|
+
if (entries.length > MAX_ENTRIES) {
|
|
2784
|
+
entries.splice(0, entries.length - MAX_ENTRIES);
|
|
2785
|
+
}
|
|
2786
|
+
}
|
|
2787
|
+
function getActivityLog(since) {
|
|
2788
|
+
if (!since) return [...entries];
|
|
2789
|
+
return entries.filter((e) => e.time > since);
|
|
2790
|
+
}
|
|
2791
|
+
function info(msg) {
|
|
2792
|
+
logActivity("info", msg);
|
|
2793
|
+
}
|
|
2794
|
+
function warn(msg) {
|
|
2795
|
+
logActivity("warn", msg);
|
|
2796
|
+
}
|
|
2797
|
+
function error(msg) {
|
|
2798
|
+
logActivity("error", msg);
|
|
2799
|
+
}
|
|
2800
|
+
|
|
2801
|
+
// src/telegram/notifier.ts
|
|
2802
|
+
import { Bot } from "grammy";
|
|
2803
|
+
|
|
2804
|
+
// src/telegram/format.ts
|
|
2805
|
+
var TELEGRAM_TEXT_LIMIT = 4096;
|
|
2806
|
+
var FENCE_OVERHEAD = 32;
|
|
2807
|
+
var TRUNCATE_BUDGET = TELEGRAM_TEXT_LIMIT - FENCE_OVERHEAD;
|
|
2808
|
+
var MAX_ROWS_INLINE = 30;
|
|
2809
|
+
var MAX_COL_WIDTH = 24;
|
|
2810
|
+
function formatResult(result, opts = {}) {
|
|
2811
|
+
const maxRows = opts.maxRows ?? MAX_ROWS_INLINE;
|
|
2812
|
+
if (Array.isArray(result)) {
|
|
2813
|
+
if (result.length === 0) return "<i>(no rows)</i>";
|
|
2814
|
+
const allObjects = result.every(
|
|
2815
|
+
(r) => r !== null && typeof r === "object" && !Array.isArray(r)
|
|
2816
|
+
);
|
|
2817
|
+
if (allObjects) {
|
|
2818
|
+
return renderTable(result, result.length, maxRows, opts);
|
|
2819
|
+
}
|
|
2820
|
+
return renderJson(result.slice(0, maxRows), result.length, maxRows);
|
|
2821
|
+
}
|
|
2822
|
+
return renderJson(result, 1, 1);
|
|
2823
|
+
}
|
|
2824
|
+
function formatError(err) {
|
|
2825
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
2826
|
+
return `\u274C <b>Error</b>
|
|
2827
|
+
<pre>${escapeHtml(msg)}</pre>`;
|
|
2828
|
+
}
|
|
2829
|
+
function renderTable(rows, totalCount, maxRows, opts) {
|
|
2830
|
+
const maxColWidth = opts.maxColWidth ?? MAX_COL_WIDTH;
|
|
2831
|
+
const sample = rows.slice(0, maxRows);
|
|
2832
|
+
const columns = [];
|
|
2833
|
+
const seen = /* @__PURE__ */ new Set();
|
|
2834
|
+
for (const r of sample) {
|
|
2835
|
+
for (const k of Object.keys(r)) {
|
|
2836
|
+
if (!seen.has(k)) {
|
|
2837
|
+
seen.add(k);
|
|
2838
|
+
columns.push(k);
|
|
2839
|
+
}
|
|
2840
|
+
}
|
|
2841
|
+
}
|
|
2842
|
+
const widths = columns.map(
|
|
2843
|
+
(c) => Math.min(
|
|
2844
|
+
maxColWidth,
|
|
2845
|
+
Math.max(c.length, ...sample.map((r) => stringifyCell(r[c]).length))
|
|
2846
|
+
)
|
|
2847
|
+
);
|
|
2848
|
+
const headerRow = columns.map((c, i) => padCell(c, widths[i])).join(" ");
|
|
2849
|
+
const sep = widths.map((w) => "\u2500".repeat(w)).join(" ");
|
|
2850
|
+
const dataRows = sample.map(
|
|
2851
|
+
(r) => columns.map((c, i) => padCell(stringifyCell(r[c]), widths[i])).join(" ")
|
|
2852
|
+
);
|
|
2853
|
+
const lines = [headerRow, sep, ...dataRows];
|
|
2854
|
+
let body = lines.join("\n");
|
|
2855
|
+
if (body.length > TRUNCATE_BUDGET) {
|
|
2856
|
+
body = body.slice(0, TRUNCATE_BUDGET - 30) + "\n... (truncated)";
|
|
2857
|
+
}
|
|
2858
|
+
let out = `<pre>${escapeHtml(body)}</pre>`;
|
|
2859
|
+
if (totalCount > maxRows) {
|
|
2860
|
+
out += `
|
|
2861
|
+
<i>showing ${maxRows} of ${totalCount} rows \u2014 narrow your query</i>`;
|
|
2862
|
+
} else {
|
|
2863
|
+
out += `
|
|
2864
|
+
<i>${totalCount} row${totalCount === 1 ? "" : "s"}</i>`;
|
|
2865
|
+
}
|
|
2866
|
+
return out;
|
|
2867
|
+
}
|
|
2868
|
+
function stringifyCell(value) {
|
|
2869
|
+
if (value === null || value === void 0) return "";
|
|
2870
|
+
if (typeof value === "string") return value;
|
|
2871
|
+
if (typeof value === "number" || typeof value === "boolean") return String(value);
|
|
2872
|
+
return JSON.stringify(value);
|
|
2873
|
+
}
|
|
2874
|
+
function padCell(s, width) {
|
|
2875
|
+
if (s.length > width) return s.slice(0, width - 1) + "\u2026";
|
|
2876
|
+
return s.padEnd(width, " ");
|
|
2877
|
+
}
|
|
2878
|
+
function renderJson(data, totalCount, maxRows) {
|
|
2879
|
+
const text = JSON.stringify(data, null, 2);
|
|
2880
|
+
let body = text;
|
|
2881
|
+
if (body.length > TRUNCATE_BUDGET) {
|
|
2882
|
+
body = body.slice(0, TRUNCATE_BUDGET - 30) + "\n... (truncated)";
|
|
2883
|
+
}
|
|
2884
|
+
let out = `<pre>${escapeHtml(body)}</pre>`;
|
|
2885
|
+
if (Array.isArray(data) && totalCount > maxRows) {
|
|
2886
|
+
out += `
|
|
2887
|
+
<i>showing ${maxRows} of ${totalCount} rows \u2014 narrow your query</i>`;
|
|
2888
|
+
}
|
|
2889
|
+
return out;
|
|
2890
|
+
}
|
|
2891
|
+
var HTML_ENTITIES = {
|
|
2892
|
+
"&": "&",
|
|
2893
|
+
"<": "<",
|
|
2894
|
+
">": ">"
|
|
2895
|
+
};
|
|
2896
|
+
function escapeHtml(s) {
|
|
2897
|
+
return s.replace(/[&<>]/g, (c) => HTML_ENTITIES[c] ?? c);
|
|
2898
|
+
}
|
|
2899
|
+
|
|
2900
|
+
// src/telegram/notifier.ts
|
|
2901
|
+
var botCache = /* @__PURE__ */ new Map();
|
|
2902
|
+
function expandEnv4(input) {
|
|
2903
|
+
return input.replace(/\$\{([A-Z0-9_]+)\}/gi, (_, name) => process.env[name] ?? "");
|
|
2904
|
+
}
|
|
2905
|
+
function getBot(token) {
|
|
2906
|
+
let bot = botCache.get(token);
|
|
2907
|
+
if (!bot) {
|
|
2908
|
+
bot = new Bot(token);
|
|
2909
|
+
botCache.set(token, bot);
|
|
2910
|
+
}
|
|
2911
|
+
return bot;
|
|
2912
|
+
}
|
|
2913
|
+
var TelegramNotifier = class {
|
|
2914
|
+
token;
|
|
2915
|
+
chatId;
|
|
2916
|
+
template;
|
|
2917
|
+
constructor(cfg) {
|
|
2918
|
+
const tokenSource = cfg.botToken ?? process.env.PIPEQUERY_TG_BOT_TOKEN;
|
|
2919
|
+
if (!tokenSource) {
|
|
2920
|
+
throw new Error(
|
|
2921
|
+
"TelegramNotifier: no bot token. Set notify.telegram.botToken or PIPEQUERY_TG_BOT_TOKEN."
|
|
2922
|
+
);
|
|
2923
|
+
}
|
|
2924
|
+
this.token = expandEnv4(tokenSource);
|
|
2925
|
+
this.chatId = cfg.chatId;
|
|
2926
|
+
this.template = cfg.message;
|
|
2927
|
+
}
|
|
2928
|
+
async notify(payload) {
|
|
2929
|
+
const bot = getBot(this.token);
|
|
2930
|
+
const text = this.renderMessage(payload);
|
|
2931
|
+
await bot.api.sendMessage(this.chatId, text, {
|
|
2932
|
+
parse_mode: "HTML",
|
|
2933
|
+
link_preview_options: { is_disabled: true }
|
|
2934
|
+
});
|
|
2935
|
+
}
|
|
2936
|
+
renderMessage(payload) {
|
|
2937
|
+
if (this.template) {
|
|
2938
|
+
return renderTemplate(this.template, payload);
|
|
2939
|
+
}
|
|
2940
|
+
const header = `\u{1F514} <b>Watch fired:</b> <code>${escapeHtml(payload.watchName)}</code>`;
|
|
2941
|
+
const reason = `<i>${escapeHtml(payload.reason)}</i>`;
|
|
2942
|
+
const body = formatResult(payload.result);
|
|
2943
|
+
return [header, reason, body].join("\n");
|
|
2944
|
+
}
|
|
2945
|
+
};
|
|
2946
|
+
function renderTemplate(tmpl, payload) {
|
|
2947
|
+
const result = payload.result;
|
|
2948
|
+
const rows = Array.isArray(result) ? result : [result];
|
|
2949
|
+
const firstRow = rows[0];
|
|
2950
|
+
const count = rows.length;
|
|
2951
|
+
return tmpl.replace(/\{\{\s*\.([a-zA-Z0-9_]+)\s*\}\}/g, (match, name) => {
|
|
2952
|
+
if (name === "count") return String(count);
|
|
2953
|
+
if (name === "watchName") return payload.watchName;
|
|
2954
|
+
if (name === "reason") return payload.reason;
|
|
2955
|
+
if (firstRow && typeof firstRow === "object" && firstRow !== null) {
|
|
2956
|
+
const value = firstRow[name];
|
|
2957
|
+
if (value === void 0) return match;
|
|
2958
|
+
return String(value);
|
|
2959
|
+
}
|
|
2960
|
+
return match;
|
|
2961
|
+
});
|
|
2962
|
+
}
|
|
2963
|
+
|
|
2964
|
+
// src/server/watches/manager.ts
|
|
2965
|
+
var DEFAULT_INTERVAL_MS4 = 6e4;
|
|
2966
|
+
var WatchManager = class {
|
|
2967
|
+
constructor(sourceManager) {
|
|
2968
|
+
this.sourceManager = sourceManager;
|
|
2969
|
+
}
|
|
2970
|
+
sourceManager;
|
|
2971
|
+
watches = /* @__PURE__ */ new Map();
|
|
2972
|
+
disposed = false;
|
|
2973
|
+
start(configs) {
|
|
2974
|
+
for (const [name, cfg] of Object.entries(configs)) {
|
|
2975
|
+
try {
|
|
2976
|
+
this.startOne(name, cfg);
|
|
2977
|
+
} catch (err) {
|
|
2978
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
2979
|
+
error(`Watch "${name}" failed to start: ${msg}`);
|
|
2980
|
+
process.stderr.write(`[pipequery-watch] "${name}" failed to start: ${msg}
|
|
2981
|
+
`);
|
|
2982
|
+
}
|
|
2983
|
+
}
|
|
2984
|
+
}
|
|
2985
|
+
startOne(name, cfg) {
|
|
2986
|
+
const notifiers = buildNotifiers(cfg.notify);
|
|
2987
|
+
if (notifiers.length === 0) {
|
|
2988
|
+
throw new Error(`watch has no notification channels configured`);
|
|
2989
|
+
}
|
|
2990
|
+
const intervalMs = cfg.interval ? parseDuration(cfg.interval) : DEFAULT_INTERVAL_MS4;
|
|
2991
|
+
const state = {
|
|
2992
|
+
config: cfg,
|
|
2993
|
+
notifiers,
|
|
2994
|
+
intervalMs,
|
|
2995
|
+
timer: null,
|
|
2996
|
+
lastWasEmpty: null,
|
|
2997
|
+
lastHash: null
|
|
2998
|
+
};
|
|
2999
|
+
this.watches.set(name, state);
|
|
3000
|
+
info(`Watch "${name}" registered (every ${cfg.interval ?? "60s"})`);
|
|
3001
|
+
void this.tick(name);
|
|
3002
|
+
state.timer = setInterval(() => void this.tick(name), intervalMs);
|
|
3003
|
+
}
|
|
3004
|
+
async tick(name) {
|
|
3005
|
+
if (this.disposed) return;
|
|
3006
|
+
const state = this.watches.get(name);
|
|
3007
|
+
if (!state) return;
|
|
3008
|
+
let result;
|
|
3009
|
+
try {
|
|
3010
|
+
const ctx = this.sourceManager.getContext();
|
|
3011
|
+
result = query(ctx, state.config.query);
|
|
3012
|
+
} catch (err) {
|
|
3013
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
3014
|
+
error(`Watch "${name}" query failed: ${msg}`);
|
|
3015
|
+
return;
|
|
3016
|
+
}
|
|
3017
|
+
const fireWhen = state.config.fireWhen ?? "when_non_empty";
|
|
3018
|
+
const isEmpty = isResultEmpty(result);
|
|
3019
|
+
const hash = hashResult(result);
|
|
3020
|
+
let shouldFire = false;
|
|
3021
|
+
let reason = "";
|
|
3022
|
+
if (fireWhen === "when_non_empty") {
|
|
3023
|
+
if (state.lastWasEmpty !== false && !isEmpty) {
|
|
3024
|
+
shouldFire = true;
|
|
3025
|
+
reason = "result transitioned to non-empty";
|
|
3026
|
+
}
|
|
3027
|
+
} else if (fireWhen === "when_empty") {
|
|
3028
|
+
if (state.lastWasEmpty !== true && isEmpty) {
|
|
3029
|
+
shouldFire = true;
|
|
3030
|
+
reason = "result transitioned to empty";
|
|
3031
|
+
}
|
|
3032
|
+
} else if (fireWhen === "on_change") {
|
|
3033
|
+
if (state.lastHash !== null && state.lastHash !== hash) {
|
|
3034
|
+
shouldFire = true;
|
|
3035
|
+
reason = "result content changed";
|
|
3036
|
+
}
|
|
3037
|
+
}
|
|
3038
|
+
state.lastWasEmpty = isEmpty;
|
|
3039
|
+
state.lastHash = hash;
|
|
3040
|
+
if (!shouldFire) return;
|
|
3041
|
+
info(`Watch "${name}" fired (${reason})`);
|
|
3042
|
+
await Promise.all(
|
|
3043
|
+
state.notifiers.map(
|
|
3044
|
+
(n) => n.notify({ watchName: name, result, reason }).catch((err) => {
|
|
3045
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
3046
|
+
error(`Watch "${name}" notify failed: ${msg}`);
|
|
3047
|
+
process.stderr.write(`[pipequery-watch] "${name}" notify failed: ${msg}
|
|
3048
|
+
`);
|
|
3049
|
+
})
|
|
3050
|
+
)
|
|
3051
|
+
);
|
|
3052
|
+
}
|
|
3053
|
+
stop(name) {
|
|
3054
|
+
const state = this.watches.get(name);
|
|
3055
|
+
if (!state) return false;
|
|
3056
|
+
if (state.timer) clearInterval(state.timer);
|
|
3057
|
+
this.watches.delete(name);
|
|
3058
|
+
return true;
|
|
3059
|
+
}
|
|
3060
|
+
list() {
|
|
3061
|
+
return [...this.watches.keys()];
|
|
3062
|
+
}
|
|
3063
|
+
dispose() {
|
|
3064
|
+
this.disposed = true;
|
|
3065
|
+
for (const state of this.watches.values()) {
|
|
3066
|
+
if (state.timer) clearInterval(state.timer);
|
|
3067
|
+
}
|
|
3068
|
+
this.watches.clear();
|
|
3069
|
+
}
|
|
3070
|
+
};
|
|
3071
|
+
function buildNotifiers(notify) {
|
|
3072
|
+
const out = [];
|
|
3073
|
+
if (notify.telegram) out.push(new TelegramNotifier(notify.telegram));
|
|
3074
|
+
return out;
|
|
3075
|
+
}
|
|
3076
|
+
function isResultEmpty(result) {
|
|
3077
|
+
if (result === null || result === void 0) return true;
|
|
3078
|
+
if (Array.isArray(result)) return result.length === 0;
|
|
3079
|
+
if (typeof result === "object") return Object.keys(result).length === 0;
|
|
3080
|
+
return false;
|
|
3081
|
+
}
|
|
3082
|
+
function hashResult(result) {
|
|
3083
|
+
const json = JSON.stringify(result, (_, v) => v === void 0 ? null : v);
|
|
3084
|
+
return createHash("sha256").update(json).digest("hex");
|
|
3085
|
+
}
|
|
3086
|
+
|
|
3087
|
+
// src/server/routes/health.ts
|
|
3088
|
+
function registerHealthRoutes(app, sourceManager, startedAt) {
|
|
3089
|
+
app.get("/health", async () => {
|
|
3090
|
+
return { status: "ok", uptime: Date.now() - startedAt.getTime() };
|
|
3091
|
+
});
|
|
3092
|
+
app.get("/status", async () => {
|
|
3093
|
+
return {
|
|
3094
|
+
status: "ok",
|
|
3095
|
+
uptime: Date.now() - startedAt.getTime(),
|
|
3096
|
+
sources: sourceManager.getAllStatuses()
|
|
3097
|
+
};
|
|
3098
|
+
});
|
|
3099
|
+
}
|
|
3100
|
+
|
|
2328
3101
|
// src/server/routes/control.ts
|
|
2329
3102
|
function registerControlRoutes(app, sourceManager, endpointRegistry, configCallbacks) {
|
|
2330
3103
|
app.get("/api/_control/sources", async () => {
|
|
@@ -2512,33 +3285,6 @@ function sendAllPanels(res, queries, sourceManager) {
|
|
|
2512
3285
|
}
|
|
2513
3286
|
}
|
|
2514
3287
|
|
|
2515
|
-
// src/server/activity-log.ts
|
|
2516
|
-
var MAX_ENTRIES = 100;
|
|
2517
|
-
var entries = [];
|
|
2518
|
-
function logActivity(level, message) {
|
|
2519
|
-
entries.push({
|
|
2520
|
-
time: (/* @__PURE__ */ new Date()).toISOString(),
|
|
2521
|
-
level,
|
|
2522
|
-
message
|
|
2523
|
-
});
|
|
2524
|
-
if (entries.length > MAX_ENTRIES) {
|
|
2525
|
-
entries.splice(0, entries.length - MAX_ENTRIES);
|
|
2526
|
-
}
|
|
2527
|
-
}
|
|
2528
|
-
function getActivityLog(since) {
|
|
2529
|
-
if (!since) return [...entries];
|
|
2530
|
-
return entries.filter((e) => e.time > since);
|
|
2531
|
-
}
|
|
2532
|
-
function info(msg) {
|
|
2533
|
-
logActivity("info", msg);
|
|
2534
|
-
}
|
|
2535
|
-
function warn(msg) {
|
|
2536
|
-
logActivity("warn", msg);
|
|
2537
|
-
}
|
|
2538
|
-
function error(msg) {
|
|
2539
|
-
logActivity("error", msg);
|
|
2540
|
-
}
|
|
2541
|
-
|
|
2542
3288
|
// src/server/app.ts
|
|
2543
3289
|
async function createServer(config, configPath, cwd) {
|
|
2544
3290
|
const app = Fastify({ logger: false });
|
|
@@ -2599,8 +3345,12 @@ async function createServer(config, configPath, cwd) {
|
|
|
2599
3345
|
registerControlRoutes(app, sourceManager, { endpoints }, configCallbacks);
|
|
2600
3346
|
registerSSERoutes(app, sourceManager);
|
|
2601
3347
|
registerDynamicRoutes(app, sourceManager, endpoints);
|
|
3348
|
+
const watchManager = new WatchManager(sourceManager);
|
|
3349
|
+
if (config.watches && Object.keys(config.watches).length > 0) {
|
|
3350
|
+
watchManager.start(config.watches);
|
|
3351
|
+
}
|
|
2602
3352
|
info("Server ready");
|
|
2603
|
-
return { app, sourceManager, endpoints };
|
|
3353
|
+
return { app, sourceManager, endpoints, watchManager };
|
|
2604
3354
|
}
|
|
2605
3355
|
|
|
2606
3356
|
// src/utils/banner.ts
|
|
@@ -2665,8 +3415,8 @@ function printServerStartup(opts) {
|
|
|
2665
3415
|
// src/commands/serve.ts
|
|
2666
3416
|
async function serveCommand(opts) {
|
|
2667
3417
|
const cwd = process.cwd();
|
|
2668
|
-
const pqDir =
|
|
2669
|
-
const serverJsonPath =
|
|
3418
|
+
const pqDir = resolve5(cwd, ".pipequery");
|
|
3419
|
+
const serverJsonPath = resolve5(pqDir, "server.json");
|
|
2670
3420
|
if (opts.stop) {
|
|
2671
3421
|
return stopDaemon(serverJsonPath);
|
|
2672
3422
|
}
|
|
@@ -2678,7 +3428,7 @@ async function serveCommand(opts) {
|
|
|
2678
3428
|
}
|
|
2679
3429
|
const { app, sourceManager } = await createServer(config, configPath, cwd);
|
|
2680
3430
|
await mkdir(pqDir, { recursive: true });
|
|
2681
|
-
const logPath =
|
|
3431
|
+
const logPath = resolve5(pqDir, "server.log");
|
|
2682
3432
|
await writeFile3(serverJsonPath, JSON.stringify({ url: `http://localhost:${port}`, pid: process.pid, daemon: false, log: logPath }));
|
|
2683
3433
|
await app.listen({ port, host });
|
|
2684
3434
|
const sourceCount = sourceManager.getSourceNames().length;
|
|
@@ -2700,9 +3450,9 @@ async function serveCommand(opts) {
|
|
|
2700
3450
|
process.on("SIGTERM", shutdown);
|
|
2701
3451
|
}
|
|
2702
3452
|
async function startDaemon(cwd, port, host) {
|
|
2703
|
-
const pqDir =
|
|
2704
|
-
const serverJsonPath =
|
|
2705
|
-
const logPath =
|
|
3453
|
+
const pqDir = resolve5(cwd, ".pipequery");
|
|
3454
|
+
const serverJsonPath = resolve5(pqDir, "server.json");
|
|
3455
|
+
const logPath = resolve5(pqDir, "server.log");
|
|
2706
3456
|
if (existsSync3(serverJsonPath)) {
|
|
2707
3457
|
try {
|
|
2708
3458
|
const info2 = JSON.parse(await readFile3(serverJsonPath, "utf-8"));
|
|
@@ -2759,11 +3509,11 @@ import chalk4 from "chalk";
|
|
|
2759
3509
|
|
|
2760
3510
|
// src/utils/server-discovery.ts
|
|
2761
3511
|
import { readFile as readFile4 } from "fs/promises";
|
|
2762
|
-
import { resolve as
|
|
3512
|
+
import { resolve as resolve6 } from "path";
|
|
2763
3513
|
import { existsSync as existsSync4 } from "fs";
|
|
2764
3514
|
async function getServerUrl() {
|
|
2765
3515
|
const cwd = process.cwd();
|
|
2766
|
-
const serverJsonPath =
|
|
3516
|
+
const serverJsonPath = resolve6(cwd, ".pipequery", "server.json");
|
|
2767
3517
|
if (existsSync4(serverJsonPath)) {
|
|
2768
3518
|
try {
|
|
2769
3519
|
const info2 = JSON.parse(await readFile4(serverJsonPath, "utf-8"));
|
|
@@ -2805,14 +3555,14 @@ async function queryCommand(expression, opts) {
|
|
|
2805
3555
|
return;
|
|
2806
3556
|
}
|
|
2807
3557
|
if (Array.isArray(result) && result.length > 0 && typeof result[0] === "object") {
|
|
2808
|
-
|
|
3558
|
+
renderTable2(result);
|
|
2809
3559
|
} else if (typeof result === "object" && result !== null && !Array.isArray(result)) {
|
|
2810
|
-
|
|
3560
|
+
renderTable2([result]);
|
|
2811
3561
|
} else {
|
|
2812
3562
|
console.log(result);
|
|
2813
3563
|
}
|
|
2814
3564
|
}
|
|
2815
|
-
function
|
|
3565
|
+
function renderTable2(rows) {
|
|
2816
3566
|
if (rows.length === 0) {
|
|
2817
3567
|
log.dim("(empty result)");
|
|
2818
3568
|
return;
|
|
@@ -2906,8 +3656,60 @@ async function sourceAddCommand(name, opts) {
|
|
|
2906
3656
|
...opts.watch ? { watch: true } : {}
|
|
2907
3657
|
};
|
|
2908
3658
|
break;
|
|
3659
|
+
case "postgres":
|
|
3660
|
+
if (!opts.url) throw new Error("--url is required for Postgres sources");
|
|
3661
|
+
if (!opts.query) throw new Error("--query is required for Postgres sources");
|
|
3662
|
+
sourceConfig = {
|
|
3663
|
+
type: "postgres",
|
|
3664
|
+
url: opts.url,
|
|
3665
|
+
query: opts.query,
|
|
3666
|
+
interval: opts.interval ?? "30s",
|
|
3667
|
+
...opts.ssl === "no-verify" ? { ssl: "no-verify" } : {},
|
|
3668
|
+
...opts.ssl === "false" ? { ssl: false } : {},
|
|
3669
|
+
...opts.maxRows ? { maxRows: Number.parseInt(opts.maxRows, 10) } : {}
|
|
3670
|
+
};
|
|
3671
|
+
break;
|
|
3672
|
+
case "mysql":
|
|
3673
|
+
if (!opts.url) throw new Error("--url is required for MySQL sources");
|
|
3674
|
+
if (!opts.query) throw new Error("--query is required for MySQL sources");
|
|
3675
|
+
sourceConfig = {
|
|
3676
|
+
type: "mysql",
|
|
3677
|
+
url: opts.url,
|
|
3678
|
+
query: opts.query,
|
|
3679
|
+
interval: opts.interval ?? "30s",
|
|
3680
|
+
...opts.ssl === "no-verify" ? { ssl: "no-verify" } : {},
|
|
3681
|
+
...opts.ssl === "false" ? { ssl: false } : {},
|
|
3682
|
+
...opts.maxRows ? { maxRows: Number.parseInt(opts.maxRows, 10) } : {}
|
|
3683
|
+
};
|
|
3684
|
+
break;
|
|
3685
|
+
case "sqlite":
|
|
3686
|
+
if (!opts.path) throw new Error("--path is required for SQLite sources");
|
|
3687
|
+
if (!opts.query) throw new Error("--query is required for SQLite sources");
|
|
3688
|
+
sourceConfig = {
|
|
3689
|
+
type: "sqlite",
|
|
3690
|
+
path: opts.path,
|
|
3691
|
+
query: opts.query,
|
|
3692
|
+
interval: opts.interval ?? "30s",
|
|
3693
|
+
...opts.maxRows ? { maxRows: Number.parseInt(opts.maxRows, 10) } : {},
|
|
3694
|
+
...opts.readonly === false ? { readonly: false } : {}
|
|
3695
|
+
};
|
|
3696
|
+
break;
|
|
3697
|
+
case "kafka":
|
|
3698
|
+
if (!opts.brokers) throw new Error("--brokers is required for Kafka sources (comma-separated)");
|
|
3699
|
+
if (!opts.topic) throw new Error("--topic is required for Kafka sources");
|
|
3700
|
+
sourceConfig = {
|
|
3701
|
+
type: "kafka",
|
|
3702
|
+
brokers: opts.brokers,
|
|
3703
|
+
topic: opts.topic,
|
|
3704
|
+
...opts.groupId ? { groupId: opts.groupId } : {},
|
|
3705
|
+
...opts.fromBeginning ? { fromBeginning: true } : {},
|
|
3706
|
+
...opts.maxBuffer ? { maxBuffer: Number.parseInt(opts.maxBuffer, 10) } : {},
|
|
3707
|
+
...opts.valueFormat ? { valueFormat: opts.valueFormat } : {},
|
|
3708
|
+
...opts.ssl === "true" ? { ssl: true } : {}
|
|
3709
|
+
};
|
|
3710
|
+
break;
|
|
2909
3711
|
default:
|
|
2910
|
-
throw new Error(`Unknown source type: ${opts.type}. Use: rest, websocket, file`);
|
|
3712
|
+
throw new Error(`Unknown source type: ${opts.type}. Use: rest, websocket, file, postgres, mysql, sqlite, kafka`);
|
|
2911
3713
|
}
|
|
2912
3714
|
config.sources[name] = sourceConfig;
|
|
2913
3715
|
await saveConfig(config, configPath);
|
|
@@ -3862,7 +4664,7 @@ async function dashboardCommand(opts) {
|
|
|
3862
4664
|
import React6 from "react";
|
|
3863
4665
|
import { render as render2 } from "ink";
|
|
3864
4666
|
import { existsSync as existsSync5, readFileSync } from "fs";
|
|
3865
|
-
import { resolve as
|
|
4667
|
+
import { resolve as resolve7 } from "path";
|
|
3866
4668
|
|
|
3867
4669
|
// src/tui/MonitorApp.tsx
|
|
3868
4670
|
import { useState as useState5, useEffect as useEffect4 } from "react";
|
|
@@ -4046,7 +4848,7 @@ function MonitorApp({ serverUrl }) {
|
|
|
4046
4848
|
// src/commands/monitor.ts
|
|
4047
4849
|
async function monitorCommand() {
|
|
4048
4850
|
const serverUrl = await getServerUrl();
|
|
4049
|
-
const discoveryPath =
|
|
4851
|
+
const discoveryPath = resolve7(process.cwd(), ".pipequery", "server.json");
|
|
4050
4852
|
let logPath;
|
|
4051
4853
|
if (existsSync5(discoveryPath)) {
|
|
4052
4854
|
try {
|
|
@@ -4058,7 +4860,7 @@ async function monitorCommand() {
|
|
|
4058
4860
|
}
|
|
4059
4861
|
}
|
|
4060
4862
|
if (!logPath) {
|
|
4061
|
-
const fallback =
|
|
4863
|
+
const fallback = resolve7(process.cwd(), ".pipequery", "server.log");
|
|
4062
4864
|
if (existsSync5(fallback)) {
|
|
4063
4865
|
logPath = fallback;
|
|
4064
4866
|
}
|
|
@@ -4078,7 +4880,7 @@ async function monitorCommand() {
|
|
|
4078
4880
|
|
|
4079
4881
|
// src/commands/remote.ts
|
|
4080
4882
|
import { writeFile as writeFile4 } from "fs/promises";
|
|
4081
|
-
import { resolve as
|
|
4883
|
+
import { resolve as resolve8 } from "path";
|
|
4082
4884
|
|
|
4083
4885
|
// src/docker/templates.ts
|
|
4084
4886
|
function generateDockerfile(port) {
|
|
@@ -4128,9 +4930,9 @@ async function remoteDeployCommand() {
|
|
|
4128
4930
|
const cwd = process.cwd();
|
|
4129
4931
|
const { config } = await loadConfig(cwd);
|
|
4130
4932
|
const port = config.server.port;
|
|
4131
|
-
await writeFile4(
|
|
4132
|
-
await writeFile4(
|
|
4133
|
-
await writeFile4(
|
|
4933
|
+
await writeFile4(resolve8(cwd, "Dockerfile"), generateDockerfile(port));
|
|
4934
|
+
await writeFile4(resolve8(cwd, "docker-compose.yaml"), generateDockerCompose(port));
|
|
4935
|
+
await writeFile4(resolve8(cwd, ".dockerignore"), generateDockerignore());
|
|
4134
4936
|
log.success("Generated Dockerfile, docker-compose.yaml, and .dockerignore");
|
|
4135
4937
|
log.info(`Run: docker compose up -d --build`);
|
|
4136
4938
|
log.info(`Then: pq remote connect http://localhost:${port}`);
|
|
@@ -4140,7 +4942,7 @@ async function loadOrCreateConfig(cwd) {
|
|
|
4140
4942
|
if (existing) {
|
|
4141
4943
|
return loadConfig(cwd);
|
|
4142
4944
|
}
|
|
4143
|
-
const configPath =
|
|
4945
|
+
const configPath = resolve8(cwd, "pipequery.yaml");
|
|
4144
4946
|
const config = {
|
|
4145
4947
|
server: { ...DEFAULT_CONFIG.server },
|
|
4146
4948
|
sources: {},
|
|
@@ -4188,6 +4990,1052 @@ async function remoteStatusCommand() {
|
|
|
4188
4990
|
}
|
|
4189
4991
|
}
|
|
4190
4992
|
|
|
4993
|
+
// src/mcp/provider.ts
|
|
4994
|
+
var LocalProvider = class {
|
|
4995
|
+
constructor(config, cwd) {
|
|
4996
|
+
this.config = config;
|
|
4997
|
+
this.sourceManager = new SourceManager(cwd);
|
|
4998
|
+
this.endpoints = new Map(Object.entries(config.endpoints));
|
|
4999
|
+
this.ready = this.bootSources();
|
|
5000
|
+
}
|
|
5001
|
+
config;
|
|
5002
|
+
sourceManager;
|
|
5003
|
+
endpoints;
|
|
5004
|
+
ready;
|
|
5005
|
+
async bootSources() {
|
|
5006
|
+
for (const [name, src] of Object.entries(this.config.sources)) {
|
|
5007
|
+
try {
|
|
5008
|
+
await this.sourceManager.addSource(name, src);
|
|
5009
|
+
} catch (err) {
|
|
5010
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
5011
|
+
process.stderr.write(`[pipequery-mcp] Source "${name}" failed to start: ${msg}
|
|
5012
|
+
`);
|
|
5013
|
+
}
|
|
5014
|
+
}
|
|
5015
|
+
}
|
|
5016
|
+
async listSources() {
|
|
5017
|
+
await this.ready;
|
|
5018
|
+
const statuses = this.sourceManager.getAllStatuses();
|
|
5019
|
+
return Object.entries(statuses).map(([name, status]) => ({ name, status }));
|
|
5020
|
+
}
|
|
5021
|
+
async describeSource(name, sampleSize = 5) {
|
|
5022
|
+
await this.ready;
|
|
5023
|
+
const status = this.sourceManager.getSourceStatus(name);
|
|
5024
|
+
const data = this.sourceManager.getSourceData(name);
|
|
5025
|
+
if (!status || !data) return void 0;
|
|
5026
|
+
const sample = data.slice(0, sampleSize);
|
|
5027
|
+
const fields = inferFields(data);
|
|
5028
|
+
return { name, status, sample, fields };
|
|
5029
|
+
}
|
|
5030
|
+
async listEndpoints() {
|
|
5031
|
+
await this.ready;
|
|
5032
|
+
return [...this.endpoints].map(([path, config]) => ({ path, config }));
|
|
5033
|
+
}
|
|
5034
|
+
async callEndpoint(path) {
|
|
5035
|
+
await this.ready;
|
|
5036
|
+
const endpoint2 = this.endpoints.get(path);
|
|
5037
|
+
if (!endpoint2) {
|
|
5038
|
+
throw new Error(`Endpoint "${path}" not found`);
|
|
5039
|
+
}
|
|
5040
|
+
const context = this.sourceManager.getContext();
|
|
5041
|
+
return query(context, endpoint2.query);
|
|
5042
|
+
}
|
|
5043
|
+
async runQuery(expression) {
|
|
5044
|
+
await this.ready;
|
|
5045
|
+
const context = this.sourceManager.getContext();
|
|
5046
|
+
return query(context, expression);
|
|
5047
|
+
}
|
|
5048
|
+
async dispose() {
|
|
5049
|
+
await this.sourceManager.dispose();
|
|
5050
|
+
}
|
|
5051
|
+
// Exposed for the HTTP plugin that wants to mount the MCP server into an
|
|
5052
|
+
// existing `createServer` instance; the caller passes in their own manager.
|
|
5053
|
+
static withSourceManager(sourceManager, endpoints) {
|
|
5054
|
+
return new AttachedLocalProvider(sourceManager, endpoints);
|
|
5055
|
+
}
|
|
5056
|
+
};
|
|
5057
|
+
var AttachedLocalProvider = class {
|
|
5058
|
+
constructor(sourceManager, endpoints) {
|
|
5059
|
+
this.sourceManager = sourceManager;
|
|
5060
|
+
this.endpoints = endpoints;
|
|
5061
|
+
}
|
|
5062
|
+
sourceManager;
|
|
5063
|
+
endpoints;
|
|
5064
|
+
async listSources() {
|
|
5065
|
+
const statuses = this.sourceManager.getAllStatuses();
|
|
5066
|
+
return Object.entries(statuses).map(([name, status]) => ({ name, status }));
|
|
5067
|
+
}
|
|
5068
|
+
async describeSource(name, sampleSize = 5) {
|
|
5069
|
+
const status = this.sourceManager.getSourceStatus(name);
|
|
5070
|
+
const data = this.sourceManager.getSourceData(name);
|
|
5071
|
+
if (!status || !data) return void 0;
|
|
5072
|
+
return { name, status, sample: data.slice(0, sampleSize), fields: inferFields(data) };
|
|
5073
|
+
}
|
|
5074
|
+
async listEndpoints() {
|
|
5075
|
+
return [...this.endpoints].map(([path, config]) => ({ path, config }));
|
|
5076
|
+
}
|
|
5077
|
+
async callEndpoint(path) {
|
|
5078
|
+
const endpoint2 = this.endpoints.get(path);
|
|
5079
|
+
if (!endpoint2) throw new Error(`Endpoint "${path}" not found`);
|
|
5080
|
+
return query(this.sourceManager.getContext(), endpoint2.query);
|
|
5081
|
+
}
|
|
5082
|
+
async runQuery(expression) {
|
|
5083
|
+
return query(this.sourceManager.getContext(), expression);
|
|
5084
|
+
}
|
|
5085
|
+
async dispose() {
|
|
5086
|
+
}
|
|
5087
|
+
};
|
|
5088
|
+
var AttachedProvider = class {
|
|
5089
|
+
constructor(baseUrl) {
|
|
5090
|
+
this.baseUrl = baseUrl;
|
|
5091
|
+
this.baseUrl = baseUrl.replace(/\/$/, "");
|
|
5092
|
+
}
|
|
5093
|
+
baseUrl;
|
|
5094
|
+
async get(path) {
|
|
5095
|
+
const res = await fetch(`${this.baseUrl}${path}`);
|
|
5096
|
+
if (!res.ok) throw new Error(`GET ${path} failed: HTTP ${res.status}`);
|
|
5097
|
+
return res.json();
|
|
5098
|
+
}
|
|
5099
|
+
async post(path, body) {
|
|
5100
|
+
const res = await fetch(`${this.baseUrl}${path}`, {
|
|
5101
|
+
method: "POST",
|
|
5102
|
+
headers: { "content-type": "application/json" },
|
|
5103
|
+
body: JSON.stringify(body)
|
|
5104
|
+
});
|
|
5105
|
+
if (!res.ok) throw new Error(`POST ${path} failed: HTTP ${res.status}`);
|
|
5106
|
+
return res.json();
|
|
5107
|
+
}
|
|
5108
|
+
async listSources() {
|
|
5109
|
+
const statuses = await this.get("/api/_control/sources");
|
|
5110
|
+
return Object.entries(statuses).map(([name, status]) => ({ name, status }));
|
|
5111
|
+
}
|
|
5112
|
+
async describeSource(name, sampleSize = 5) {
|
|
5113
|
+
const res = await this.post(`/api/_control/sources/${encodeURIComponent(name)}/test`, {});
|
|
5114
|
+
if (!res.ok || !res.status || !res.sample) return void 0;
|
|
5115
|
+
return {
|
|
5116
|
+
name,
|
|
5117
|
+
status: res.status,
|
|
5118
|
+
sample: res.sample.slice(0, sampleSize),
|
|
5119
|
+
fields: inferFields(res.sample)
|
|
5120
|
+
};
|
|
5121
|
+
}
|
|
5122
|
+
async listEndpoints() {
|
|
5123
|
+
const endpoints = await this.get("/api/_control/endpoints");
|
|
5124
|
+
return Object.entries(endpoints).map(([path, config]) => ({ path, config }));
|
|
5125
|
+
}
|
|
5126
|
+
async callEndpoint(path) {
|
|
5127
|
+
const normalized = path.startsWith("/") ? path : `/${path}`;
|
|
5128
|
+
const res = await fetch(`${this.baseUrl}${normalized}`);
|
|
5129
|
+
if (!res.ok) throw new Error(`GET ${normalized} failed: HTTP ${res.status}`);
|
|
5130
|
+
return res.json();
|
|
5131
|
+
}
|
|
5132
|
+
async runQuery(expression) {
|
|
5133
|
+
const res = await this.post(
|
|
5134
|
+
"/api/_control/query",
|
|
5135
|
+
{ query: expression }
|
|
5136
|
+
);
|
|
5137
|
+
if (!res.ok) throw new Error(res.error ?? "Query failed");
|
|
5138
|
+
return res.result;
|
|
5139
|
+
}
|
|
5140
|
+
async dispose() {
|
|
5141
|
+
}
|
|
5142
|
+
};
|
|
5143
|
+
function inferFields(rows) {
|
|
5144
|
+
const fields = /* @__PURE__ */ new Set();
|
|
5145
|
+
for (const row of rows.slice(0, 20)) {
|
|
5146
|
+
if (row && typeof row === "object" && !Array.isArray(row)) {
|
|
5147
|
+
for (const k of Object.keys(row)) fields.add(k);
|
|
5148
|
+
}
|
|
5149
|
+
}
|
|
5150
|
+
return [...fields];
|
|
5151
|
+
}
|
|
5152
|
+
|
|
5153
|
+
// src/mcp/stdio.ts
|
|
5154
|
+
import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
|
|
5155
|
+
|
|
5156
|
+
// src/mcp/server.ts
|
|
5157
|
+
import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
5158
|
+
import { z } from "zod";
|
|
5159
|
+
var DEFAULT_INFO = {
|
|
5160
|
+
name: "pipequery",
|
|
5161
|
+
version: "0.1.0"
|
|
5162
|
+
};
|
|
5163
|
+
function createMcpServer(provider, info2 = DEFAULT_INFO) {
|
|
5164
|
+
const mcp2 = new McpServer(info2);
|
|
5165
|
+
mcp2.registerTool(
|
|
5166
|
+
"query",
|
|
5167
|
+
{
|
|
5168
|
+
title: "Run a PipeQuery expression",
|
|
5169
|
+
description: "Execute a PipeQuery pipe-based expression (e.g. `crypto | where(price > 100) | sort(price desc) | first(5)`) against all currently configured sources. Returns the computed result as JSON.",
|
|
5170
|
+
inputSchema: {
|
|
5171
|
+
expression: z.string().min(1).describe("The PipeQuery expression. Source names used in the expression must be configured; call `list_sources` to see them.")
|
|
5172
|
+
}
|
|
5173
|
+
},
|
|
5174
|
+
async ({ expression }) => {
|
|
5175
|
+
const result = await provider.runQuery(expression);
|
|
5176
|
+
return toolResult(result);
|
|
5177
|
+
}
|
|
5178
|
+
);
|
|
5179
|
+
mcp2.registerTool(
|
|
5180
|
+
"list_sources",
|
|
5181
|
+
{
|
|
5182
|
+
title: "List configured data sources",
|
|
5183
|
+
description: "List every data source configured in pipequery.yaml along with its current health, row count, and last-fetch timestamp."
|
|
5184
|
+
},
|
|
5185
|
+
async () => {
|
|
5186
|
+
const sources = await provider.listSources();
|
|
5187
|
+
return toolResult({ count: sources.length, sources });
|
|
5188
|
+
}
|
|
5189
|
+
);
|
|
5190
|
+
mcp2.registerTool(
|
|
5191
|
+
"describe_source",
|
|
5192
|
+
{
|
|
5193
|
+
title: "Describe a data source",
|
|
5194
|
+
description: "Return a sample of rows and the inferred field names for a single source. Useful before constructing a query.",
|
|
5195
|
+
inputSchema: {
|
|
5196
|
+
name: z.string().min(1).describe("Source name as returned by list_sources."),
|
|
5197
|
+
sample_size: z.number().int().positive().max(100).optional().describe("Number of rows to include in the sample. Defaults to 5, max 100.")
|
|
5198
|
+
}
|
|
5199
|
+
},
|
|
5200
|
+
async ({ name, sample_size }) => {
|
|
5201
|
+
const description = await provider.describeSource(name, sample_size);
|
|
5202
|
+
if (!description) {
|
|
5203
|
+
return toolError(`Source "${name}" not found`);
|
|
5204
|
+
}
|
|
5205
|
+
return toolResult(description);
|
|
5206
|
+
}
|
|
5207
|
+
);
|
|
5208
|
+
mcp2.registerTool(
|
|
5209
|
+
"list_endpoints",
|
|
5210
|
+
{
|
|
5211
|
+
title: "List pre-configured query endpoints",
|
|
5212
|
+
description: "Return every endpoint declared in pipequery.yaml along with its stored query expression. These endpoints can be invoked via `call_endpoint`."
|
|
5213
|
+
},
|
|
5214
|
+
async () => {
|
|
5215
|
+
const endpoints = await provider.listEndpoints();
|
|
5216
|
+
return toolResult({ count: endpoints.length, endpoints });
|
|
5217
|
+
}
|
|
5218
|
+
);
|
|
5219
|
+
mcp2.registerTool(
|
|
5220
|
+
"call_endpoint",
|
|
5221
|
+
{
|
|
5222
|
+
title: "Invoke a pre-configured endpoint",
|
|
5223
|
+
description: "Execute the stored query for an endpoint path (e.g. `/api/top-coins`) and return its JSON result. Equivalent to the HTTP endpoint on `pq serve`, without needing a live HTTP call.",
|
|
5224
|
+
inputSchema: {
|
|
5225
|
+
path: z.string().min(1).describe("Endpoint path, exactly as declared in pipequery.yaml (typically begins with `/`).")
|
|
5226
|
+
}
|
|
5227
|
+
},
|
|
5228
|
+
async ({ path }) => {
|
|
5229
|
+
try {
|
|
5230
|
+
const result = await provider.callEndpoint(path);
|
|
5231
|
+
return toolResult(result);
|
|
5232
|
+
} catch (err) {
|
|
5233
|
+
return toolError(err instanceof Error ? err.message : String(err));
|
|
5234
|
+
}
|
|
5235
|
+
}
|
|
5236
|
+
);
|
|
5237
|
+
return mcp2;
|
|
5238
|
+
}
|
|
5239
|
+
function toolResult(data) {
|
|
5240
|
+
return {
|
|
5241
|
+
content: [{ type: "text", text: JSON.stringify(data, null, 2) }]
|
|
5242
|
+
};
|
|
5243
|
+
}
|
|
5244
|
+
function toolError(message) {
|
|
5245
|
+
return {
|
|
5246
|
+
content: [{ type: "text", text: message }],
|
|
5247
|
+
isError: true
|
|
5248
|
+
};
|
|
5249
|
+
}
|
|
5250
|
+
|
|
5251
|
+
// src/mcp/stdio.ts
|
|
5252
|
+
async function runStdio(provider, info2) {
|
|
5253
|
+
const mcp2 = createMcpServer(provider, info2);
|
|
5254
|
+
const transport = new StdioServerTransport();
|
|
5255
|
+
await mcp2.connect(transport);
|
|
5256
|
+
const cleanup = async () => {
|
|
5257
|
+
try {
|
|
5258
|
+
await mcp2.close();
|
|
5259
|
+
} finally {
|
|
5260
|
+
await provider.dispose();
|
|
5261
|
+
}
|
|
5262
|
+
};
|
|
5263
|
+
process.stdin.on("close", () => {
|
|
5264
|
+
void cleanup().then(() => process.exit(0));
|
|
5265
|
+
});
|
|
5266
|
+
process.on("SIGINT", () => void cleanup().then(() => process.exit(0)));
|
|
5267
|
+
process.on("SIGTERM", () => void cleanup().then(() => process.exit(0)));
|
|
5268
|
+
}
|
|
5269
|
+
|
|
5270
|
+
// src/mcp/http.ts
|
|
5271
|
+
import { createServer as createHttpServer } from "http";
|
|
5272
|
+
import { randomUUID } from "crypto";
|
|
5273
|
+
import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/streamableHttp.js";
|
|
5274
|
+
async function runHttp(provider, opts) {
|
|
5275
|
+
const host = opts.host ?? "127.0.0.1";
|
|
5276
|
+
const port = opts.port;
|
|
5277
|
+
const mountPath = opts.path ?? "/mcp";
|
|
5278
|
+
const transports = /* @__PURE__ */ new Map();
|
|
5279
|
+
const authToken = opts.authToken;
|
|
5280
|
+
if (!authToken) {
|
|
5281
|
+
process.stderr.write(
|
|
5282
|
+
"[pipequery-mcp] WARNING: HTTP endpoint is UNAUTHENTICATED. Set --auth-token or PIPEQUERY_MCP_TOKEN for anything beyond localhost.\n"
|
|
5283
|
+
);
|
|
5284
|
+
}
|
|
5285
|
+
const attachTransport = async () => {
|
|
5286
|
+
const transport = new StreamableHTTPServerTransport({
|
|
5287
|
+
sessionIdGenerator: () => randomUUID(),
|
|
5288
|
+
onsessioninitialized: (sessionId) => {
|
|
5289
|
+
transports.set(sessionId, transport);
|
|
5290
|
+
},
|
|
5291
|
+
onsessionclosed: (sessionId) => {
|
|
5292
|
+
transports.delete(sessionId);
|
|
5293
|
+
}
|
|
5294
|
+
});
|
|
5295
|
+
const mcp2 = createMcpServer(provider, opts.info);
|
|
5296
|
+
await mcp2.connect(transport);
|
|
5297
|
+
return transport;
|
|
5298
|
+
};
|
|
5299
|
+
const server = createHttpServer(async (req, res) => {
|
|
5300
|
+
if (!req.url) {
|
|
5301
|
+
res.statusCode = 400;
|
|
5302
|
+
res.end("Bad Request");
|
|
5303
|
+
return;
|
|
5304
|
+
}
|
|
5305
|
+
const requestPath = req.url.split("?")[0];
|
|
5306
|
+
if (requestPath !== mountPath) {
|
|
5307
|
+
res.statusCode = 404;
|
|
5308
|
+
res.end("Not Found");
|
|
5309
|
+
return;
|
|
5310
|
+
}
|
|
5311
|
+
if (authToken && !isAuthorized(req, authToken)) {
|
|
5312
|
+
res.statusCode = 401;
|
|
5313
|
+
res.setHeader("content-type", "application/json");
|
|
5314
|
+
res.setHeader("www-authenticate", 'Bearer realm="pipequery-mcp"');
|
|
5315
|
+
res.end(
|
|
5316
|
+
JSON.stringify({
|
|
5317
|
+
jsonrpc: "2.0",
|
|
5318
|
+
error: { code: -32001, message: "Unauthorized: missing or invalid bearer token" },
|
|
5319
|
+
id: null
|
|
5320
|
+
})
|
|
5321
|
+
);
|
|
5322
|
+
return;
|
|
5323
|
+
}
|
|
5324
|
+
try {
|
|
5325
|
+
const body = await readJsonBody(req);
|
|
5326
|
+
const sessionId = getSessionId(req);
|
|
5327
|
+
let transport = sessionId ? transports.get(sessionId) : void 0;
|
|
5328
|
+
if (!transport) {
|
|
5329
|
+
transport = await attachTransport();
|
|
5330
|
+
}
|
|
5331
|
+
await transport.handleRequest(req, res, body);
|
|
5332
|
+
} catch (err) {
|
|
5333
|
+
process.stderr.write(
|
|
5334
|
+
`[pipequery-mcp] transport error: ${err instanceof Error ? err.stack ?? err.message : String(err)}
|
|
5335
|
+
`
|
|
5336
|
+
);
|
|
5337
|
+
if (!res.headersSent) {
|
|
5338
|
+
res.statusCode = 500;
|
|
5339
|
+
res.setHeader("content-type", "application/json");
|
|
5340
|
+
res.end(JSON.stringify({
|
|
5341
|
+
jsonrpc: "2.0",
|
|
5342
|
+
error: {
|
|
5343
|
+
code: -32603,
|
|
5344
|
+
message: err instanceof Error ? err.message : String(err)
|
|
5345
|
+
},
|
|
5346
|
+
id: null
|
|
5347
|
+
}));
|
|
5348
|
+
} else {
|
|
5349
|
+
res.end();
|
|
5350
|
+
}
|
|
5351
|
+
}
|
|
5352
|
+
});
|
|
5353
|
+
await new Promise((resolve10) => server.listen(port, host, resolve10));
|
|
5354
|
+
return {
|
|
5355
|
+
url: `http://${host}:${port}${mountPath}`,
|
|
5356
|
+
async close() {
|
|
5357
|
+
await new Promise((resolve10, reject) => {
|
|
5358
|
+
server.close((err) => err ? reject(err) : resolve10());
|
|
5359
|
+
});
|
|
5360
|
+
for (const t of transports.values()) {
|
|
5361
|
+
await t.close();
|
|
5362
|
+
}
|
|
5363
|
+
transports.clear();
|
|
5364
|
+
await provider.dispose();
|
|
5365
|
+
}
|
|
5366
|
+
};
|
|
5367
|
+
}
|
|
5368
|
+
function isAuthorized(req, expected) {
|
|
5369
|
+
const header = req.headers["authorization"];
|
|
5370
|
+
const raw = typeof header === "string" ? header : Array.isArray(header) ? header[0] : void 0;
|
|
5371
|
+
if (!raw) return false;
|
|
5372
|
+
const match = raw.match(/^Bearer\s+(.+)$/i);
|
|
5373
|
+
if (!match) return false;
|
|
5374
|
+
const token = match[1].trim();
|
|
5375
|
+
return timingSafeEqual(token, expected);
|
|
5376
|
+
}
|
|
5377
|
+
function timingSafeEqual(a, b) {
|
|
5378
|
+
if (a.length !== b.length) return false;
|
|
5379
|
+
let diff = 0;
|
|
5380
|
+
for (let i = 0; i < a.length; i++) diff |= a.charCodeAt(i) ^ b.charCodeAt(i);
|
|
5381
|
+
return diff === 0;
|
|
5382
|
+
}
|
|
5383
|
+
function getSessionId(req) {
|
|
5384
|
+
const header = req.headers["mcp-session-id"];
|
|
5385
|
+
if (typeof header === "string") return header;
|
|
5386
|
+
if (Array.isArray(header)) return header[0];
|
|
5387
|
+
return void 0;
|
|
5388
|
+
}
|
|
5389
|
+
async function readJsonBody(req) {
|
|
5390
|
+
const method = req.method?.toUpperCase();
|
|
5391
|
+
if (method === "GET" || method === "DELETE") return void 0;
|
|
5392
|
+
const chunks = [];
|
|
5393
|
+
for await (const chunk of req) {
|
|
5394
|
+
chunks.push(chunk);
|
|
5395
|
+
}
|
|
5396
|
+
if (chunks.length === 0) return void 0;
|
|
5397
|
+
const text = Buffer.concat(chunks).toString("utf8");
|
|
5398
|
+
if (!text.trim()) return void 0;
|
|
5399
|
+
try {
|
|
5400
|
+
return JSON.parse(text);
|
|
5401
|
+
} catch {
|
|
5402
|
+
throw new Error("Invalid JSON body");
|
|
5403
|
+
}
|
|
5404
|
+
}
|
|
5405
|
+
|
|
5406
|
+
// src/commands/mcp.ts
|
|
5407
|
+
import { Client } from "@modelcontextprotocol/sdk/client/index.js";
|
|
5408
|
+
import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js";
|
|
5409
|
+
async function mcpServeCommand(opts) {
|
|
5410
|
+
const provider = await buildProvider(opts.attach);
|
|
5411
|
+
const isStdio = !opts.http;
|
|
5412
|
+
if (isStdio) {
|
|
5413
|
+
await runStdio(provider);
|
|
5414
|
+
return;
|
|
5415
|
+
}
|
|
5416
|
+
const port = opts.port ?? 3001;
|
|
5417
|
+
const host = opts.host ?? "127.0.0.1";
|
|
5418
|
+
const authToken = opts.authToken ?? process.env.PIPEQUERY_MCP_TOKEN;
|
|
5419
|
+
const handle = await runHttp(provider, { port, host, authToken });
|
|
5420
|
+
log.success(`PipeQuery MCP server listening on ${handle.url}`);
|
|
5421
|
+
log.info("Connect any MCP client to the URL above (Claude, Cursor, custom agents).");
|
|
5422
|
+
log.info("Press Ctrl+C to stop.");
|
|
5423
|
+
const shutdown = async () => {
|
|
5424
|
+
log.info("Shutting down MCP server...");
|
|
5425
|
+
await handle.close();
|
|
5426
|
+
process.exit(0);
|
|
5427
|
+
};
|
|
5428
|
+
process.on("SIGINT", () => void shutdown());
|
|
5429
|
+
process.on("SIGTERM", () => void shutdown());
|
|
5430
|
+
}
|
|
5431
|
+
async function mcpInspectCommand(opts) {
|
|
5432
|
+
const provider = await buildProvider(opts.attach);
|
|
5433
|
+
const mcp2 = createMcpServer(provider);
|
|
5434
|
+
const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair();
|
|
5435
|
+
await mcp2.connect(serverTransport);
|
|
5436
|
+
const client = new Client({ name: "pq-mcp-inspect", version: "0.1.0" });
|
|
5437
|
+
await client.connect(clientTransport);
|
|
5438
|
+
try {
|
|
5439
|
+
const { tools } = await client.listTools();
|
|
5440
|
+
const summary = {
|
|
5441
|
+
server: "pipequery",
|
|
5442
|
+
toolCount: tools.length,
|
|
5443
|
+
tools: tools.map((t) => ({
|
|
5444
|
+
name: t.name,
|
|
5445
|
+
title: t.title,
|
|
5446
|
+
description: t.description,
|
|
5447
|
+
inputSchema: t.inputSchema
|
|
5448
|
+
}))
|
|
5449
|
+
};
|
|
5450
|
+
process.stdout.write(JSON.stringify(summary, null, 2) + "\n");
|
|
5451
|
+
} finally {
|
|
5452
|
+
await client.close();
|
|
5453
|
+
await mcp2.close();
|
|
5454
|
+
await provider.dispose();
|
|
5455
|
+
}
|
|
5456
|
+
}
|
|
5457
|
+
async function buildProvider(attachUrl) {
|
|
5458
|
+
if (attachUrl) {
|
|
5459
|
+
return new AttachedProvider(attachUrl);
|
|
5460
|
+
}
|
|
5461
|
+
const cwd = process.cwd();
|
|
5462
|
+
const { config } = await loadConfig(cwd);
|
|
5463
|
+
return new LocalProvider(config, cwd);
|
|
5464
|
+
}
|
|
5465
|
+
|
|
5466
|
+
// src/telegram/bot.ts
|
|
5467
|
+
import { Bot as Bot2, GrammyError, HttpError } from "grammy";
|
|
5468
|
+
function buildBot(token, provider, opts = {}) {
|
|
5469
|
+
const bot = new Bot2(token);
|
|
5470
|
+
const allowList = normalizeAllowList(opts.allowUsers ?? []);
|
|
5471
|
+
const nl = opts.nl;
|
|
5472
|
+
bot.use(async (ctx, next) => {
|
|
5473
|
+
if (allowList.size === 0) {
|
|
5474
|
+
const id = ctx.from?.id;
|
|
5475
|
+
if (id) trackUnauthSeen(id, ctx.from?.username);
|
|
5476
|
+
await next();
|
|
5477
|
+
return;
|
|
5478
|
+
}
|
|
5479
|
+
if (!isAllowed(ctx, allowList)) {
|
|
5480
|
+
await ctx.reply("\u{1F512} You are not authorized to use this bot.");
|
|
5481
|
+
return;
|
|
5482
|
+
}
|
|
5483
|
+
await next();
|
|
5484
|
+
});
|
|
5485
|
+
bot.command("start", async (ctx) => {
|
|
5486
|
+
await ctx.reply(
|
|
5487
|
+
[
|
|
5488
|
+
"\u{1F44B} <b>pipequery bot</b>",
|
|
5489
|
+
"",
|
|
5490
|
+
"Run pipequery expressions against your configured data sources from chat.",
|
|
5491
|
+
"",
|
|
5492
|
+
"<b>Commands</b>",
|
|
5493
|
+
"/sources \u2014 list configured sources and their health",
|
|
5494
|
+
"/describe <name> \u2014 sample rows + field names",
|
|
5495
|
+
"/endpoints \u2014 list pre-configured endpoints",
|
|
5496
|
+
"/call <path> \u2014 execute a pre-configured endpoint",
|
|
5497
|
+
"/query <expression> \u2014 run an arbitrary pipequery expression",
|
|
5498
|
+
"/help \u2014 this message"
|
|
5499
|
+
].join("\n"),
|
|
5500
|
+
{ parse_mode: "HTML" }
|
|
5501
|
+
);
|
|
5502
|
+
});
|
|
5503
|
+
bot.command("help", async (ctx) => {
|
|
5504
|
+
const lines = [
|
|
5505
|
+
"<b>Examples</b>",
|
|
5506
|
+
"<code>/sources</code>",
|
|
5507
|
+
"<code>/describe products</code>",
|
|
5508
|
+
"<code>/query products | sort(price desc) | first(5)</code>",
|
|
5509
|
+
"<code>/query crypto | where(price > 100) | rollup(avg(price) as avg)</code>",
|
|
5510
|
+
"<code>/call /api/top-coins</code>"
|
|
5511
|
+
];
|
|
5512
|
+
if (nl) {
|
|
5513
|
+
lines.push(
|
|
5514
|
+
"",
|
|
5515
|
+
"<b>Natural language</b>",
|
|
5516
|
+
"Send any plain-text message (no leading <code>/</code>) and I'll translate it into a pipequery expression and run it.",
|
|
5517
|
+
'Example: <i>"top 5 most expensive paid orders"</i>'
|
|
5518
|
+
);
|
|
5519
|
+
}
|
|
5520
|
+
lines.push(
|
|
5521
|
+
"",
|
|
5522
|
+
"<b>Tips</b>",
|
|
5523
|
+
"\u2022 Wrap quoted strings carefully \u2014 Telegram's autocorrect can mangle them. Send long expressions as a separate message.",
|
|
5524
|
+
"\u2022 Use <code>/describe <source></code> to discover field names before constructing a where clause.",
|
|
5525
|
+
"\u2022 Results > 30 rows are truncated with a footer; tighten your query to see all of them."
|
|
5526
|
+
);
|
|
5527
|
+
await ctx.reply(lines.join("\n"), { parse_mode: "HTML" });
|
|
5528
|
+
});
|
|
5529
|
+
bot.command("sources", async (ctx) => {
|
|
5530
|
+
try {
|
|
5531
|
+
const sources = await provider.listSources();
|
|
5532
|
+
if (sources.length === 0) {
|
|
5533
|
+
await ctx.reply("<i>No sources configured.</i>", { parse_mode: "HTML" });
|
|
5534
|
+
return;
|
|
5535
|
+
}
|
|
5536
|
+
const lines = sources.map((s) => {
|
|
5537
|
+
const dot = s.status.healthy ? "\u{1F7E2}" : "\u{1F534}";
|
|
5538
|
+
const rowCount = s.status.rowCount.toLocaleString();
|
|
5539
|
+
const err = s.status.error ? ` <i>(${escapeHtml(s.status.error)})</i>` : "";
|
|
5540
|
+
return `${dot} <b>${escapeHtml(s.name)}</b> \u2014 ${rowCount} rows${err}`;
|
|
5541
|
+
});
|
|
5542
|
+
await ctx.reply(lines.join("\n"), { parse_mode: "HTML" });
|
|
5543
|
+
} catch (err) {
|
|
5544
|
+
await ctx.reply(formatError(err), { parse_mode: "HTML" });
|
|
5545
|
+
}
|
|
5546
|
+
});
|
|
5547
|
+
bot.command("describe", async (ctx) => {
|
|
5548
|
+
const name = ctx.match.trim();
|
|
5549
|
+
if (!name) {
|
|
5550
|
+
await ctx.reply("Usage: <code>/describe <source-name></code>", { parse_mode: "HTML" });
|
|
5551
|
+
return;
|
|
5552
|
+
}
|
|
5553
|
+
try {
|
|
5554
|
+
const desc = await provider.describeSource(name, 5);
|
|
5555
|
+
if (!desc) {
|
|
5556
|
+
await ctx.reply(`Source <code>${escapeHtml(name)}</code> not found.`, { parse_mode: "HTML" });
|
|
5557
|
+
return;
|
|
5558
|
+
}
|
|
5559
|
+
const fields = desc.fields.length ? desc.fields.map((f) => `<code>${escapeHtml(f)}</code>`).join(", ") : "<i>(none inferred)</i>";
|
|
5560
|
+
const sample = formatResult(desc.sample);
|
|
5561
|
+
await ctx.reply(
|
|
5562
|
+
`<b>${escapeHtml(name)}</b> \u2014 ${desc.status.rowCount.toLocaleString()} rows
|
|
5563
|
+
<b>Fields:</b> ${fields}
|
|
5564
|
+
|
|
5565
|
+
<b>Sample:</b>
|
|
5566
|
+
${sample}`,
|
|
5567
|
+
{ parse_mode: "HTML" }
|
|
5568
|
+
);
|
|
5569
|
+
} catch (err) {
|
|
5570
|
+
await ctx.reply(formatError(err), { parse_mode: "HTML" });
|
|
5571
|
+
}
|
|
5572
|
+
});
|
|
5573
|
+
bot.command("endpoints", async (ctx) => {
|
|
5574
|
+
try {
|
|
5575
|
+
const endpoints = await provider.listEndpoints();
|
|
5576
|
+
if (endpoints.length === 0) {
|
|
5577
|
+
await ctx.reply("<i>No endpoints configured.</i>", { parse_mode: "HTML" });
|
|
5578
|
+
return;
|
|
5579
|
+
}
|
|
5580
|
+
const lines = endpoints.map(
|
|
5581
|
+
(e) => `<b>${escapeHtml(e.path)}</b>
|
|
5582
|
+
<code>${escapeHtml(e.config.query)}</code>`
|
|
5583
|
+
);
|
|
5584
|
+
await ctx.reply(lines.join("\n\n"), { parse_mode: "HTML" });
|
|
5585
|
+
} catch (err) {
|
|
5586
|
+
await ctx.reply(formatError(err), { parse_mode: "HTML" });
|
|
5587
|
+
}
|
|
5588
|
+
});
|
|
5589
|
+
bot.command("call", async (ctx) => {
|
|
5590
|
+
const path = ctx.match.trim();
|
|
5591
|
+
if (!path) {
|
|
5592
|
+
await ctx.reply("Usage: <code>/call <endpoint-path></code>", { parse_mode: "HTML" });
|
|
5593
|
+
return;
|
|
5594
|
+
}
|
|
5595
|
+
try {
|
|
5596
|
+
const result = await provider.callEndpoint(path);
|
|
5597
|
+
await ctx.reply(formatResult(result), { parse_mode: "HTML" });
|
|
5598
|
+
} catch (err) {
|
|
5599
|
+
await ctx.reply(formatError(err), { parse_mode: "HTML" });
|
|
5600
|
+
}
|
|
5601
|
+
});
|
|
5602
|
+
bot.command("query", async (ctx) => {
|
|
5603
|
+
const expr = ctx.match.trim();
|
|
5604
|
+
if (!expr) {
|
|
5605
|
+
await ctx.reply("Usage: <code>/query <pipequery expression></code>", { parse_mode: "HTML" });
|
|
5606
|
+
return;
|
|
5607
|
+
}
|
|
5608
|
+
try {
|
|
5609
|
+
const result = await provider.runQuery(expr);
|
|
5610
|
+
await ctx.reply(formatResult(result), { parse_mode: "HTML" });
|
|
5611
|
+
} catch (err) {
|
|
5612
|
+
await ctx.reply(formatError(err), { parse_mode: "HTML" });
|
|
5613
|
+
}
|
|
5614
|
+
});
|
|
5615
|
+
bot.on("message:text", async (ctx) => {
|
|
5616
|
+
const text = ctx.message.text.trim();
|
|
5617
|
+
if (!text || text.startsWith("/")) return;
|
|
5618
|
+
if (!nl) {
|
|
5619
|
+
await ctx.reply(
|
|
5620
|
+
"I don't recognise that. Use <code>/help</code> to see the available commands. Natural-language queries are disabled \u2014 start the bot with <code>--anthropic-key</code> to enable them.",
|
|
5621
|
+
{ parse_mode: "HTML" }
|
|
5622
|
+
);
|
|
5623
|
+
return;
|
|
5624
|
+
}
|
|
5625
|
+
try {
|
|
5626
|
+
await ctx.replyWithChatAction("typing");
|
|
5627
|
+
const { expression, explanation } = await nl.translate(text);
|
|
5628
|
+
if (!expression) {
|
|
5629
|
+
const reason = explanation || "I couldn't translate that into a pipequery expression.";
|
|
5630
|
+
await ctx.reply(`\u{1F914} ${escapeHtml(reason)}`, { parse_mode: "HTML" });
|
|
5631
|
+
return;
|
|
5632
|
+
}
|
|
5633
|
+
const result = await provider.runQuery(expression);
|
|
5634
|
+
const header = `<i>${escapeHtml(explanation || "Translated query")}</i>
|
|
5635
|
+
<code>${escapeHtml(expression)}</code>
|
|
5636
|
+
|
|
5637
|
+
`;
|
|
5638
|
+
await ctx.reply(header + formatResult(result), { parse_mode: "HTML" });
|
|
5639
|
+
} catch (err) {
|
|
5640
|
+
await ctx.reply(formatError(err), { parse_mode: "HTML" });
|
|
5641
|
+
}
|
|
5642
|
+
});
|
|
5643
|
+
bot.catch((err) => {
|
|
5644
|
+
if (err.error instanceof GrammyError) {
|
|
5645
|
+
process.stderr.write(`[pipequery-telegram] grammy error: ${err.error.description}
|
|
5646
|
+
`);
|
|
5647
|
+
} else if (err.error instanceof HttpError) {
|
|
5648
|
+
process.stderr.write(`[pipequery-telegram] http error: ${err.error.message}
|
|
5649
|
+
`);
|
|
5650
|
+
} else {
|
|
5651
|
+
process.stderr.write(`[pipequery-telegram] unhandled: ${err.error}
|
|
5652
|
+
`);
|
|
5653
|
+
}
|
|
5654
|
+
});
|
|
5655
|
+
return bot;
|
|
5656
|
+
}
|
|
5657
|
+
function normalizeAllowList(entries2) {
|
|
5658
|
+
const set = /* @__PURE__ */ new Set();
|
|
5659
|
+
for (const raw of entries2) {
|
|
5660
|
+
const trimmed = raw.trim().replace(/^@/, "").toLowerCase();
|
|
5661
|
+
if (trimmed) set.add(trimmed);
|
|
5662
|
+
}
|
|
5663
|
+
return set;
|
|
5664
|
+
}
|
|
5665
|
+
function isAllowed(ctx, allowList) {
|
|
5666
|
+
const from = ctx.from;
|
|
5667
|
+
if (!from) return false;
|
|
5668
|
+
if (allowList.has(String(from.id))) return true;
|
|
5669
|
+
if (from.username && allowList.has(from.username.toLowerCase())) return true;
|
|
5670
|
+
return false;
|
|
5671
|
+
}
|
|
5672
|
+
var seenUnauth = /* @__PURE__ */ new Set();
|
|
5673
|
+
function trackUnauthSeen(id, username) {
|
|
5674
|
+
if (seenUnauth.has(id)) return;
|
|
5675
|
+
seenUnauth.add(id);
|
|
5676
|
+
process.stderr.write(
|
|
5677
|
+
`[pipequery-telegram] WARNING: bot has no allowlist; user "${username ?? id}" can query freely. Restart with --allow-user @yourname to lock it down.
|
|
5678
|
+
`
|
|
5679
|
+
);
|
|
5680
|
+
}
|
|
5681
|
+
|
|
5682
|
+
// src/telegram/nl.ts
|
|
5683
|
+
import Anthropic from "@anthropic-ai/sdk";
|
|
5684
|
+
var MODEL = "claude-haiku-4-5";
|
|
5685
|
+
var MAX_TOKENS = 1024;
|
|
5686
|
+
var SCHEMA_TTL_MS = 6e4;
|
|
5687
|
+
function createNLTranslator(provider, opts) {
|
|
5688
|
+
const client = opts.client ?? new Anthropic({ apiKey: opts.apiKey });
|
|
5689
|
+
const model = opts.model ?? MODEL;
|
|
5690
|
+
let cachedSchema = null;
|
|
5691
|
+
async function getSchemaPreamble() {
|
|
5692
|
+
const now = Date.now();
|
|
5693
|
+
if (cachedSchema && now - cachedSchema.builtAt < SCHEMA_TTL_MS) {
|
|
5694
|
+
return cachedSchema.text;
|
|
5695
|
+
}
|
|
5696
|
+
const text = await buildSchemaPreamble(provider);
|
|
5697
|
+
cachedSchema = { text, builtAt: now };
|
|
5698
|
+
return text;
|
|
5699
|
+
}
|
|
5700
|
+
async function translate(question) {
|
|
5701
|
+
const schema = await getSchemaPreamble();
|
|
5702
|
+
const response = await client.messages.create({
|
|
5703
|
+
model,
|
|
5704
|
+
max_tokens: MAX_TOKENS,
|
|
5705
|
+
system: [
|
|
5706
|
+
// Cache breakpoint #1 — the always-stable DSL grammar + instructions.
|
|
5707
|
+
// This is the largest block and the one that benefits most from being
|
|
5708
|
+
// cached across every translation request the bot ever serves.
|
|
5709
|
+
{
|
|
5710
|
+
type: "text",
|
|
5711
|
+
text: SYSTEM_PROMPT,
|
|
5712
|
+
cache_control: { type: "ephemeral" }
|
|
5713
|
+
},
|
|
5714
|
+
// Cache breakpoint #2 — the per-tenant source schema. Stable until
|
|
5715
|
+
// the yaml changes (or SCHEMA_TTL_MS elapses).
|
|
5716
|
+
{
|
|
5717
|
+
type: "text",
|
|
5718
|
+
text: schema,
|
|
5719
|
+
cache_control: { type: "ephemeral" }
|
|
5720
|
+
}
|
|
5721
|
+
],
|
|
5722
|
+
messages: [
|
|
5723
|
+
{
|
|
5724
|
+
role: "user",
|
|
5725
|
+
content: question
|
|
5726
|
+
}
|
|
5727
|
+
]
|
|
5728
|
+
});
|
|
5729
|
+
const text = extractText(response);
|
|
5730
|
+
return parseTranslation(text);
|
|
5731
|
+
}
|
|
5732
|
+
return {
|
|
5733
|
+
translate,
|
|
5734
|
+
invalidateSchema: () => {
|
|
5735
|
+
cachedSchema = null;
|
|
5736
|
+
}
|
|
5737
|
+
};
|
|
5738
|
+
}
|
|
5739
|
+
async function buildSchemaPreamble(provider) {
|
|
5740
|
+
const sources = await provider.listSources();
|
|
5741
|
+
const endpoints = await provider.listEndpoints();
|
|
5742
|
+
const sourceLines = [];
|
|
5743
|
+
for (const s of sources) {
|
|
5744
|
+
let fields = [];
|
|
5745
|
+
try {
|
|
5746
|
+
const desc = await provider.describeSource(s.name, 3);
|
|
5747
|
+
if (desc) fields = desc.fields;
|
|
5748
|
+
} catch {
|
|
5749
|
+
}
|
|
5750
|
+
const fieldStr = fields.length ? ` \u2014 fields: ${fields.join(", ")}` : "";
|
|
5751
|
+
sourceLines.push(`- ${s.name} (${s.status.rowCount} rows)${fieldStr}`);
|
|
5752
|
+
}
|
|
5753
|
+
const endpointLines = endpoints.map((e) => `- ${e.path} \u2192 ${e.config.query}`);
|
|
5754
|
+
return [
|
|
5755
|
+
"## Available data sources",
|
|
5756
|
+
sourceLines.length ? sourceLines.join("\n") : "(no sources configured)",
|
|
5757
|
+
"",
|
|
5758
|
+
"## Pre-defined endpoints",
|
|
5759
|
+
endpointLines.length ? endpointLines.join("\n") : "(no endpoints configured)"
|
|
5760
|
+
].join("\n");
|
|
5761
|
+
}
|
|
5762
|
+
function extractText(response) {
|
|
5763
|
+
const parts = [];
|
|
5764
|
+
for (const block of response.content) {
|
|
5765
|
+
if (block.type === "text") parts.push(block.text);
|
|
5766
|
+
}
|
|
5767
|
+
return parts.join("").trim();
|
|
5768
|
+
}
|
|
5769
|
+
function parseTranslation(raw) {
|
|
5770
|
+
const cleaned = raw.replace(/^```json\s*/i, "").replace(/^```\s*/i, "").replace(/```\s*$/i, "").trim();
|
|
5771
|
+
const start = cleaned.indexOf("{");
|
|
5772
|
+
const end = cleaned.lastIndexOf("}");
|
|
5773
|
+
if (start === -1 || end === -1 || end < start) {
|
|
5774
|
+
throw new Error(`Translator returned non-JSON: ${truncate(raw, 200)}`);
|
|
5775
|
+
}
|
|
5776
|
+
const slice = cleaned.slice(start, end + 1);
|
|
5777
|
+
let parsed;
|
|
5778
|
+
try {
|
|
5779
|
+
parsed = JSON.parse(slice);
|
|
5780
|
+
} catch (err) {
|
|
5781
|
+
throw new Error(`Translator returned invalid JSON: ${truncate(raw, 200)}`);
|
|
5782
|
+
}
|
|
5783
|
+
if (typeof parsed !== "object" || parsed === null || typeof parsed.expression !== "string") {
|
|
5784
|
+
throw new Error(`Translator response missing "expression" field: ${truncate(raw, 200)}`);
|
|
5785
|
+
}
|
|
5786
|
+
const obj = parsed;
|
|
5787
|
+
return {
|
|
5788
|
+
expression: obj.expression.trim(),
|
|
5789
|
+
explanation: typeof obj.explanation === "string" ? obj.explanation.trim() : ""
|
|
5790
|
+
};
|
|
5791
|
+
}
|
|
5792
|
+
function truncate(s, n) {
|
|
5793
|
+
return s.length > n ? `${s.slice(0, n)}\u2026` : s;
|
|
5794
|
+
}
|
|
5795
|
+
var SYSTEM_PROMPT = `You are a translator that converts natural-language questions into PipeQuery expressions.
|
|
5796
|
+
|
|
5797
|
+
PipeQuery is a small, pipe-style query DSL. Every expression starts with a source name (or a literal array) and pipes into one or more operations using \`|\`. PipeQuery is read-only by construction \u2014 there is no DDL, DML, INSERT, UPDATE, DELETE, or DROP. You will never produce SQL.
|
|
5798
|
+
|
|
5799
|
+
# Output contract
|
|
5800
|
+
|
|
5801
|
+
You MUST respond with a SINGLE JSON object and nothing else. No prose, no markdown, no code fences.
|
|
5802
|
+
|
|
5803
|
+
The object has exactly two fields:
|
|
5804
|
+
- "expression": a string containing the PipeQuery expression to execute.
|
|
5805
|
+
- "explanation": a one-sentence plain-English description of what the expression does.
|
|
5806
|
+
|
|
5807
|
+
Example:
|
|
5808
|
+
{"expression": "orders | where(status == \\"paid\\") | sort(total desc) | first(5)", "explanation": "Top 5 paid orders by total."}
|
|
5809
|
+
|
|
5810
|
+
If the user's question cannot be answered with the available sources, return:
|
|
5811
|
+
{"expression": "", "explanation": "<brief reason why no expression was produced>"}
|
|
5812
|
+
|
|
5813
|
+
# Grammar
|
|
5814
|
+
|
|
5815
|
+
Pipeline:
|
|
5816
|
+
<source_name> | <operation> | <operation> | ...
|
|
5817
|
+
|
|
5818
|
+
Sources are referenced by their bare name (no quotes). The available source names are listed in the schema preamble that follows this prompt.
|
|
5819
|
+
|
|
5820
|
+
# Operations
|
|
5821
|
+
|
|
5822
|
+
## where(<expression>)
|
|
5823
|
+
Filter rows. The expression is evaluated per row and must return a boolean.
|
|
5824
|
+
- Comparison operators: == != > >= < <=
|
|
5825
|
+
- Logical operators: && || !
|
|
5826
|
+
- Arithmetic: + - * / %
|
|
5827
|
+
- Field reference: bare identifier (e.g., \`price\`, \`status\`)
|
|
5828
|
+
- Strings: double-quoted ("paid")
|
|
5829
|
+
- Numbers: 1, 1.5, -42
|
|
5830
|
+
- Booleans: true, false
|
|
5831
|
+
- Null: null
|
|
5832
|
+
|
|
5833
|
+
Examples:
|
|
5834
|
+
events | where(severity == "high")
|
|
5835
|
+
trades | where(price > 100 && symbol == "BTC")
|
|
5836
|
+
users | where(!banned)
|
|
5837
|
+
|
|
5838
|
+
## sort(<field> asc|desc, ...)
|
|
5839
|
+
Sort by one or more fields. Direction is optional, defaults to asc.
|
|
5840
|
+
orders | sort(total desc)
|
|
5841
|
+
orders | sort(country asc, total desc)
|
|
5842
|
+
|
|
5843
|
+
## first(N) / last(N)
|
|
5844
|
+
Take the first / last N rows. N must be a positive integer literal.
|
|
5845
|
+
trades | sort(ts desc) | first(10)
|
|
5846
|
+
|
|
5847
|
+
## distinct / distinct(field, ...)
|
|
5848
|
+
Deduplicate rows. With no args, dedupes on the full row; with args, dedupes on the listed fields.
|
|
5849
|
+
users | distinct(email)
|
|
5850
|
+
|
|
5851
|
+
## select(<expression>, ...)
|
|
5852
|
+
Project fields. Each expression can be a bare field, a computed expression, or an alias (\`expr as name\`).
|
|
5853
|
+
orders | select(id, total, total * 0.1 as tax)
|
|
5854
|
+
|
|
5855
|
+
## groupBy(<field>, ...) and rollup
|
|
5856
|
+
Aggregate. \`groupBy\` keeps groups; \`rollup\` collapses each group to one row with computed aggregates.
|
|
5857
|
+
|
|
5858
|
+
orders | rollup(country, sum(total) as revenue, count() as n)
|
|
5859
|
+
trades | groupBy(symbol) | rollup(avg(price) as avg)
|
|
5860
|
+
|
|
5861
|
+
Aggregate functions: sum, avg, min, max, count, median, stddev, var, percentile, distinct_count, first_value, last_value, vwap, wavg, pct, sharpe, calmar, sortino, info_ratio, drawdown, sum_abs, abs_sum.
|
|
5862
|
+
|
|
5863
|
+
\`count()\` takes no field. \`percentile(field, p)\` and \`vwap(value, weight)\` take two args.
|
|
5864
|
+
|
|
5865
|
+
## flatten / flatten(field)
|
|
5866
|
+
Unnest array values. With \`flatten(orders)\` each element of \`row.orders\` becomes its own row.
|
|
5867
|
+
|
|
5868
|
+
## map(<expression>, ...)
|
|
5869
|
+
Transform each row into a new shape (similar to \`select\` but always emits a new object).
|
|
5870
|
+
|
|
5871
|
+
## reduce(initial, accumulator)
|
|
5872
|
+
Fold rows into a single value. Rare; prefer rollup.
|
|
5873
|
+
|
|
5874
|
+
## join(<source>, <condition>)
|
|
5875
|
+
Inner join with another source on a boolean condition. Use \`left\` and \`right\` to refer to fields.
|
|
5876
|
+
orders | join(customers, left.customer_id == right.id)
|
|
5877
|
+
|
|
5878
|
+
## pivot(<field>, <agg>, ...)
|
|
5879
|
+
Pivot rows into columns by a key field.
|
|
5880
|
+
|
|
5881
|
+
# Conventions and best practices
|
|
5882
|
+
|
|
5883
|
+
1. Prefer \`rollup\` over \`groupBy\` unless the caller wants the per-group rows kept.
|
|
5884
|
+
2. Always cap result size with \`first(N)\` for "top N" or "show me" questions; default to 10 if the user doesn't specify.
|
|
5885
|
+
3. Use the EXACT field names from the schema preamble. Do not invent fields.
|
|
5886
|
+
4. If two sources need to be joined and there's no obvious join key in the schema, prefer to ask for clarification by emitting the empty-expression form rather than guessing.
|
|
5887
|
+
5. Quote string literals with double quotes; escape internal double quotes with \\".
|
|
5888
|
+
6. Do not produce SQL, JavaScript, or English in the expression field \u2014 only PipeQuery.
|
|
5889
|
+
7. Numbers are bare; no thousands separators, no quotes.
|
|
5890
|
+
8. Comparisons against null use \`== null\` / \`!= null\`.
|
|
5891
|
+
|
|
5892
|
+
# Worked examples
|
|
5893
|
+
|
|
5894
|
+
Question: "What are the top 5 most expensive orders that were paid?"
|
|
5895
|
+
{"expression": "orders | where(status == \\"paid\\") | sort(total desc) | first(5)", "explanation": "Top 5 paid orders by total."}
|
|
5896
|
+
|
|
5897
|
+
Question: "Average BTC price right now"
|
|
5898
|
+
{"expression": "trades | where(symbol == \\"BTC\\") | rollup(avg(price) as avg_price)", "explanation": "Average price across all BTC rows."}
|
|
5899
|
+
|
|
5900
|
+
Question: "Revenue by country"
|
|
5901
|
+
{"expression": "orders | rollup(country, sum(total) as revenue) | sort(revenue desc)", "explanation": "Sum of order totals grouped by country, highest first."}
|
|
5902
|
+
|
|
5903
|
+
Question: "How many distinct users placed an order?"
|
|
5904
|
+
{"expression": "orders | rollup(distinct_count(user_id) as users)", "explanation": "Count of unique user IDs in orders."}
|
|
5905
|
+
|
|
5906
|
+
Question: "Show high-severity events from the last hour"
|
|
5907
|
+
(only answerable if the schema has a timestamp field; otherwise return empty.)
|
|
5908
|
+
{"expression": "events | where(severity == \\"high\\") | sort(ts desc) | first(50)", "explanation": "Most recent high-severity events, capped at 50."}
|
|
5909
|
+
|
|
5910
|
+
Question: "Drop the orders table"
|
|
5911
|
+
{"expression": "", "explanation": "PipeQuery is read-only and cannot drop tables."}
|
|
5912
|
+
|
|
5913
|
+
Question: "How many rows in foo?"
|
|
5914
|
+
(only answerable if a source named \`foo\` exists; otherwise return empty.)
|
|
5915
|
+
{"expression": "foo | rollup(count() as n)", "explanation": "Row count of foo."}
|
|
5916
|
+
|
|
5917
|
+
# Reminder
|
|
5918
|
+
|
|
5919
|
+
- Output exactly ONE JSON object.
|
|
5920
|
+
- The "expression" field must be valid PipeQuery referencing only sources from the schema preamble.
|
|
5921
|
+
- Use double quotes for strings, escape inner quotes as \\".
|
|
5922
|
+
- Default result cap is first(10) for unbounded "show me" questions unless an aggregation makes the result naturally small.
|
|
5923
|
+
- Never produce SQL or destructive operations.
|
|
5924
|
+
`;
|
|
5925
|
+
|
|
5926
|
+
// src/commands/telegram.ts
|
|
5927
|
+
async function telegramServeCommand(opts) {
|
|
5928
|
+
const token = opts.botToken ?? process.env.PIPEQUERY_TG_BOT_TOKEN;
|
|
5929
|
+
if (!token) {
|
|
5930
|
+
log.error(
|
|
5931
|
+
"Bot token required. Pass --bot-token <token> or set PIPEQUERY_TG_BOT_TOKEN."
|
|
5932
|
+
);
|
|
5933
|
+
process.exit(1);
|
|
5934
|
+
}
|
|
5935
|
+
const anthropicKey = opts.anthropicKey ?? process.env.ANTHROPIC_API_KEY;
|
|
5936
|
+
const provider = await buildProvider2(opts.attach);
|
|
5937
|
+
const nl = anthropicKey ? createNLTranslator(provider, { apiKey: anthropicKey }) : void 0;
|
|
5938
|
+
const bot = buildBot(token, provider, {
|
|
5939
|
+
allowUsers: opts.allowUser ?? [],
|
|
5940
|
+
nl
|
|
5941
|
+
});
|
|
5942
|
+
const me = await bot.api.getMe();
|
|
5943
|
+
log.success(`Telegram bot listening as @${me.username} (id ${me.id})`);
|
|
5944
|
+
if (nl) {
|
|
5945
|
+
log.info("Natural-language translation enabled (claude-haiku-4-5).");
|
|
5946
|
+
} else {
|
|
5947
|
+
log.dim("Natural-language translation disabled. Pass --anthropic-key or set ANTHROPIC_API_KEY to enable.");
|
|
5948
|
+
}
|
|
5949
|
+
if (!opts.allowUser || opts.allowUser.length === 0) {
|
|
5950
|
+
log.warn("No --allow-user set: anyone with the bot username can query. Add allowlist for production.");
|
|
5951
|
+
}
|
|
5952
|
+
log.info("Press Ctrl+C to stop.");
|
|
5953
|
+
const shutdown = async () => {
|
|
5954
|
+
log.info("Stopping bot...");
|
|
5955
|
+
await bot.stop();
|
|
5956
|
+
await provider.dispose();
|
|
5957
|
+
process.exit(0);
|
|
5958
|
+
};
|
|
5959
|
+
process.on("SIGINT", () => void shutdown());
|
|
5960
|
+
process.on("SIGTERM", () => void shutdown());
|
|
5961
|
+
await bot.start();
|
|
5962
|
+
}
|
|
5963
|
+
async function buildProvider2(attachUrl) {
|
|
5964
|
+
if (attachUrl) {
|
|
5965
|
+
return new AttachedProvider(attachUrl);
|
|
5966
|
+
}
|
|
5967
|
+
const cwd = process.cwd();
|
|
5968
|
+
const { config } = await loadConfig(cwd);
|
|
5969
|
+
return new LocalProvider(config, cwd);
|
|
5970
|
+
}
|
|
5971
|
+
|
|
5972
|
+
// src/commands/watch.ts
|
|
5973
|
+
import chalk7 from "chalk";
|
|
5974
|
+
async function watchListCommand() {
|
|
5975
|
+
const cwd = process.cwd();
|
|
5976
|
+
const { config } = await loadConfig(cwd);
|
|
5977
|
+
const watches = config.watches ?? {};
|
|
5978
|
+
const entries2 = Object.entries(watches);
|
|
5979
|
+
if (entries2.length === 0) {
|
|
5980
|
+
log.dim("No watches configured. Add one with: pq watch add <name>");
|
|
5981
|
+
return;
|
|
5982
|
+
}
|
|
5983
|
+
console.log(chalk7.bold("Watches:\n"));
|
|
5984
|
+
for (const [name, w] of entries2) {
|
|
5985
|
+
const interval = w.interval ?? "60s";
|
|
5986
|
+
const fireWhen = w.fireWhen ?? "when_non_empty";
|
|
5987
|
+
const channels = Object.keys(w.notify).join(", ") || "<none>";
|
|
5988
|
+
console.log(` ${chalk7.bold(name)} ${chalk7.dim(`every ${interval} \u2022 ${fireWhen}`)}`);
|
|
5989
|
+
console.log(` ${chalk7.dim("query: ")}${w.query}`);
|
|
5990
|
+
console.log(` ${chalk7.dim("notify: ")}${channels}`);
|
|
5991
|
+
}
|
|
5992
|
+
}
|
|
5993
|
+
async function watchAddCommand(name, opts) {
|
|
5994
|
+
const cwd = process.cwd();
|
|
5995
|
+
const { config, path: configPath } = await loadConfig(cwd);
|
|
5996
|
+
if (!opts.telegramChatId) {
|
|
5997
|
+
throw new Error("--telegram-chat-id is required (only Telegram notifier supported in v1)");
|
|
5998
|
+
}
|
|
5999
|
+
const fireWhen = opts.fireWhen ?? "when_non_empty";
|
|
6000
|
+
if (!["when_non_empty", "when_empty", "on_change"].includes(fireWhen)) {
|
|
6001
|
+
throw new Error(`--fire-when must be one of: when_non_empty, when_empty, on_change`);
|
|
6002
|
+
}
|
|
6003
|
+
const watch3 = {
|
|
6004
|
+
query: opts.query,
|
|
6005
|
+
...opts.interval ? { interval: opts.interval } : {},
|
|
6006
|
+
fireWhen,
|
|
6007
|
+
notify: {
|
|
6008
|
+
telegram: {
|
|
6009
|
+
chatId: parseChatId(opts.telegramChatId),
|
|
6010
|
+
...opts.telegramBotToken ? { botToken: opts.telegramBotToken } : {},
|
|
6011
|
+
...opts.telegramMessage ? { message: opts.telegramMessage } : {}
|
|
6012
|
+
}
|
|
6013
|
+
}
|
|
6014
|
+
};
|
|
6015
|
+
config.watches = config.watches ?? {};
|
|
6016
|
+
config.watches[name] = watch3;
|
|
6017
|
+
await saveConfig(config, configPath);
|
|
6018
|
+
log.success(`Watch "${name}" added. Restart \`pq serve\` to activate.`);
|
|
6019
|
+
}
|
|
6020
|
+
async function watchRemoveCommand(name) {
|
|
6021
|
+
const cwd = process.cwd();
|
|
6022
|
+
const { config, path: configPath } = await loadConfig(cwd);
|
|
6023
|
+
if (!config.watches?.[name]) {
|
|
6024
|
+
log.error(`Watch "${name}" not found`);
|
|
6025
|
+
return;
|
|
6026
|
+
}
|
|
6027
|
+
delete config.watches[name];
|
|
6028
|
+
await saveConfig(config, configPath);
|
|
6029
|
+
log.success(`Watch "${name}" removed. Restart \`pq serve\` to apply.`);
|
|
6030
|
+
}
|
|
6031
|
+
function parseChatId(input) {
|
|
6032
|
+
if (/^-?\d+$/.test(input)) {
|
|
6033
|
+
const n = Number(input);
|
|
6034
|
+
if (Number.isFinite(n) && Number.isSafeInteger(n)) return n;
|
|
6035
|
+
}
|
|
6036
|
+
return input;
|
|
6037
|
+
}
|
|
6038
|
+
|
|
4191
6039
|
// src/commands/completion.ts
|
|
4192
6040
|
var BASH_COMPLETION = `###-begin-pq-completions-###
|
|
4193
6041
|
_pq_completions() {
|
|
@@ -4352,11 +6200,11 @@ function detectShell() {
|
|
|
4352
6200
|
|
|
4353
6201
|
// src/commands/stop.ts
|
|
4354
6202
|
import { readFile as readFile5, unlink as unlink2 } from "fs/promises";
|
|
4355
|
-
import { resolve as
|
|
6203
|
+
import { resolve as resolve9 } from "path";
|
|
4356
6204
|
import { existsSync as existsSync6 } from "fs";
|
|
4357
6205
|
async function stopCommand(opts) {
|
|
4358
6206
|
const cwd = process.cwd();
|
|
4359
|
-
const serverJsonPath =
|
|
6207
|
+
const serverJsonPath = resolve9(cwd, ".pipequery", "server.json");
|
|
4360
6208
|
if (!existsSync6(serverJsonPath)) {
|
|
4361
6209
|
log.error("No running server found.");
|
|
4362
6210
|
log.dim("Start one with: pq serve or pq serve -d");
|
|
@@ -4403,7 +6251,7 @@ async function stopCommand(opts) {
|
|
|
4403
6251
|
// src/commands/repl.ts
|
|
4404
6252
|
import { createInterface } from "readline";
|
|
4405
6253
|
import { spawnSync } from "child_process";
|
|
4406
|
-
import
|
|
6254
|
+
import chalk8 from "chalk";
|
|
4407
6255
|
var COMMANDS = {
|
|
4408
6256
|
help: "Show available commands",
|
|
4409
6257
|
serve: "Start the server (serve -d for daemon, serve --stop to stop)",
|
|
@@ -4421,15 +6269,15 @@ async function startRepl() {
|
|
|
4421
6269
|
let serverUrl = null;
|
|
4422
6270
|
try {
|
|
4423
6271
|
serverUrl = await getServerUrl();
|
|
4424
|
-
log.success(`Connected to server at ${
|
|
6272
|
+
log.success(`Connected to server at ${chalk8.white.bold(serverUrl)}`);
|
|
4425
6273
|
} catch {
|
|
4426
6274
|
log.warn("No running server found. Start one with: pq serve");
|
|
4427
6275
|
}
|
|
4428
|
-
console.log(
|
|
6276
|
+
console.log(chalk8.dim(' Type "help" for available commands, "exit" to quit.\n'));
|
|
4429
6277
|
const rl = createInterface({
|
|
4430
6278
|
input: process.stdin,
|
|
4431
6279
|
output: process.stdout,
|
|
4432
|
-
prompt:
|
|
6280
|
+
prompt: chalk8.hex("#06b6d4").bold(" pq > "),
|
|
4433
6281
|
terminal: true
|
|
4434
6282
|
});
|
|
4435
6283
|
rl.prompt();
|
|
@@ -4446,7 +6294,7 @@ async function startRepl() {
|
|
|
4446
6294
|
case "help":
|
|
4447
6295
|
console.log();
|
|
4448
6296
|
for (const [name, desc] of Object.entries(COMMANDS)) {
|
|
4449
|
-
console.log(` ${
|
|
6297
|
+
console.log(` ${chalk8.hex("#06b6d4")(name.padEnd(12))} ${chalk8.dim(desc)}`);
|
|
4450
6298
|
}
|
|
4451
6299
|
console.log();
|
|
4452
6300
|
break;
|
|
@@ -4493,7 +6341,7 @@ async function startRepl() {
|
|
|
4493
6341
|
});
|
|
4494
6342
|
try {
|
|
4495
6343
|
serverUrl = await getServerUrl();
|
|
4496
|
-
log.success(`Connected to server at ${
|
|
6344
|
+
log.success(`Connected to server at ${chalk8.white.bold(serverUrl)}`);
|
|
4497
6345
|
} catch {
|
|
4498
6346
|
}
|
|
4499
6347
|
rl.resume();
|
|
@@ -4521,7 +6369,7 @@ async function startRepl() {
|
|
|
4521
6369
|
break;
|
|
4522
6370
|
case "exit":
|
|
4523
6371
|
case "quit":
|
|
4524
|
-
console.log(
|
|
6372
|
+
console.log(chalk8.dim(" Goodbye.\n"));
|
|
4525
6373
|
rl.close();
|
|
4526
6374
|
process.exit(0);
|
|
4527
6375
|
break;
|
|
@@ -4565,20 +6413,20 @@ function printTable(rows) {
|
|
|
4565
6413
|
const maxData = rows.reduce((max, row) => Math.max(max, formatCell3(row[col]).length), 0);
|
|
4566
6414
|
return Math.max(col.length, maxData);
|
|
4567
6415
|
});
|
|
4568
|
-
const header = columns.map((col, i) =>
|
|
4569
|
-
const separator = widths.map((w) =>
|
|
6416
|
+
const header = columns.map((col, i) => chalk8.hex("#06b6d4").bold(col.padEnd(widths[i]))).join(" ");
|
|
6417
|
+
const separator = widths.map((w) => chalk8.dim("\u2500".repeat(w))).join(chalk8.dim("\u2500\u2500"));
|
|
4570
6418
|
console.log();
|
|
4571
6419
|
console.log(` ${header}`);
|
|
4572
6420
|
console.log(` ${separator}`);
|
|
4573
6421
|
for (const row of rows) {
|
|
4574
6422
|
const line = columns.map((col, i) => {
|
|
4575
6423
|
const val = formatCell3(row[col]);
|
|
4576
|
-
return typeof row[col] === "number" ?
|
|
6424
|
+
return typeof row[col] === "number" ? chalk8.hex("#f59e0b")(val.padStart(widths[i])) : val.padEnd(widths[i]);
|
|
4577
6425
|
}).join(" ");
|
|
4578
6426
|
console.log(` ${line}`);
|
|
4579
6427
|
}
|
|
4580
6428
|
console.log();
|
|
4581
|
-
console.log(
|
|
6429
|
+
console.log(chalk8.dim(` ${rows.length} row(s)`));
|
|
4582
6430
|
console.log();
|
|
4583
6431
|
}
|
|
4584
6432
|
function formatCell3(value) {
|
|
@@ -4596,19 +6444,20 @@ async function fetchAndPrint(serverUrl, path, key) {
|
|
|
4596
6444
|
}
|
|
4597
6445
|
|
|
4598
6446
|
// src/index.ts
|
|
6447
|
+
var collectRepeated = (value, acc) => [...acc, value];
|
|
4599
6448
|
var program = new Command();
|
|
4600
6449
|
program.name("pq").description("PipeQuery CLI \u2014 data sources, aggregations, APIs, and terminal dashboards").version("0.1.0").addHelpText("beforeAll", () => {
|
|
4601
6450
|
printBanner();
|
|
4602
6451
|
return "";
|
|
4603
6452
|
}).configureHelp({
|
|
4604
|
-
subcommandTerm: (cmd) =>
|
|
6453
|
+
subcommandTerm: (cmd) => chalk9.hex("#06b6d4")(cmd.name()) + (cmd.usage() ? " " + chalk9.dim(cmd.usage()) : "")
|
|
4605
6454
|
});
|
|
4606
6455
|
program.command("init").description("Create a pipequery.yaml config file").action(initCommand);
|
|
4607
6456
|
program.command("serve").description("Start the PipeQuery server").option("-p, --port <port>", "Server port", parseInt).option("-H, --host <host>", "Server host").option("-d, --daemon", "Run as a background daemon").option("--stop", "Stop a running daemon").action(serveCommand);
|
|
4608
6457
|
program.command("query <expression>").description("Run an ad-hoc PipeQuery expression").option("-f, --format <format>", "Output format: table, json", "table").action(queryCommand);
|
|
4609
6458
|
var source = program.command("source").description("Manage data sources");
|
|
4610
6459
|
source.command("list").description("List configured data sources").action(sourceListCommand);
|
|
4611
|
-
source.command("add <name>").description("Add a data source").requiredOption("-t, --type <type>", "Source type: rest, websocket, file").option("-u, --url <url>", "URL for REST
|
|
6460
|
+
source.command("add <name>").description("Add a data source").requiredOption("-t, --type <type>", "Source type: rest, websocket, file, postgres, mysql, sqlite, kafka").option("-u, --url <url>", "Connection URL for REST / WebSocket / Postgres / MySQL sources").option("-i, --interval <interval>", "Poll interval (e.g., 30s, 5m)", "30s").option("-d, --data-path <path>", "Dot-path to extract data from REST response").option("-p, --path <path>", "File path for file or sqlite sources").option("-w, --watch", "Watch file for changes (file sources)").option("-q, --query <sql>", "SELECT query for Postgres / MySQL / SQLite sources").option("--ssl <mode>", "Postgres / MySQL SSL: require (default), no-verify, false; Kafka: true / false").option("--max-rows <n>", "DB sources: safety cap on rows per fetch (default 10000)").option("--no-readonly", "SQLite: open the database read-write instead of read-only").option("-b, --brokers <list>", "Kafka: comma-separated bootstrap brokers (supports ${ENV_VAR})").option("--topic <topic>", "Kafka topic to subscribe to").option("--group-id <id>", "Kafka consumer group (default: ephemeral per-process)").option("--from-beginning", "Kafka: read topic from the start instead of latest").option("--value-format <fmt>", "Kafka value decoding: json (default), string, raw").option("--max-buffer <n>", "Kafka / WebSocket: ring buffer size (default 1000)").action(sourceAddCommand);
|
|
4612
6461
|
source.command("remove <name>").description("Remove a data source").action(sourceRemoveCommand);
|
|
4613
6462
|
source.command("test <name>").description("Test a data source and show sample data").action(sourceTestCommand);
|
|
4614
6463
|
var endpoint = program.command("endpoint").description("Manage API endpoints");
|
|
@@ -4621,6 +6470,15 @@ var remote = program.command("remote").description("Manage remote deployment");
|
|
|
4621
6470
|
remote.command("deploy").description("Generate Dockerfile and docker-compose.yaml").action(remoteDeployCommand);
|
|
4622
6471
|
remote.command("connect <url>").description("Connect CLI to a remote PipeQuery server").action(remoteConnectCommand);
|
|
4623
6472
|
remote.command("status").description("Check remote server health").action(remoteStatusCommand);
|
|
6473
|
+
var mcp = program.command("mcp").description("Model Context Protocol server for AI clients");
|
|
6474
|
+
mcp.command("serve").description("Start an MCP server exposing pipequery as tools (for Claude, Cursor, Copilot, etc.)").option("--http", "Serve over HTTP/SSE instead of stdio").option("-p, --port <port>", "HTTP port (when --http is set)", parseInt).option("-H, --host <host>", "HTTP host (when --http is set)").option("--attach <url>", "Attach to a running `pq serve` instance at the given URL instead of loading pipequery.yaml locally").option("--auth-token <token>", "Require Authorization: Bearer <token> on HTTP requests. Falls back to $PIPEQUERY_MCP_TOKEN. Without either, the endpoint is unauthenticated.").action(mcpServeCommand);
|
|
6475
|
+
mcp.command("inspect").description("Print the MCP tool schemas as JSON (useful for directory submission / debugging)").option("--attach <url>", "Attach to a running `pq serve` instance at the given URL instead of loading pipequery.yaml locally").action(mcpInspectCommand);
|
|
6476
|
+
var telegram = program.command("telegram").description("Telegram bot transport for pipequery");
|
|
6477
|
+
telegram.command("serve").description("Run a Telegram bot that exposes pipequery commands (/query, /sources, /describe, /endpoints, /call) plus natural-language queries when --anthropic-key is set").option("-t, --bot-token <token>", "Telegram bot token (or set $PIPEQUERY_TG_BOT_TOKEN)").option("--attach <url>", "Attach to a running `pq serve` instance instead of loading pipequery.yaml locally").option("--allow-user <handle>", "Allowlist a user (numeric id or @username); repeatable. With no allowlist, anyone with the bot username can query.", collectRepeated, []).option("--anthropic-key <key>", "Anthropic API key (or set $ANTHROPIC_API_KEY) \u2014 enables natural-language \u2192 pipequery translation on plain-text messages via claude-haiku-4-5.").action(telegramServeCommand);
|
|
6478
|
+
var watch2 = program.command("watch").description("Manage alert watches (query \u2192 notification when condition fires)");
|
|
6479
|
+
watch2.command("list").description("List configured watches").action(watchListCommand);
|
|
6480
|
+
watch2.command("add <name>").description("Add a watch that fires a notification when its query result triggers the configured condition").requiredOption("-q, --query <expression>", "PipeQuery expression to evaluate on each tick").option("-i, --interval <duration>", "Poll interval (e.g., 30s, 5m). Default 60s.").option("-f, --fire-when <mode>", "Fire condition: when_non_empty (default), when_empty, on_change").option("--telegram-chat-id <id>", "Telegram chat / channel / group ID to notify").option("--telegram-bot-token <token>", "Bot token override (defaults to $PIPEQUERY_TG_BOT_TOKEN)").option("--telegram-message <template>", "Optional message template; supports {{ .field }} from the first row, plus {{ .count }}, {{ .watchName }}, {{ .reason }}").action(watchAddCommand);
|
|
6481
|
+
watch2.command("remove <name>").description("Remove a watch").action(watchRemoveCommand);
|
|
4624
6482
|
program.command("stop").description("Stop the running PipeQuery server (daemon or foreground)").option("-f, --force", "Force kill (SIGKILL instead of SIGTERM)").action(stopCommand);
|
|
4625
6483
|
program.command("completion").description("Generate shell completion script").option("-s, --shell <shell>", "Shell type: bash, zsh, fish").action(completionCommand);
|
|
4626
6484
|
if (process.argv.length <= 2) {
|