@tradejs/infra 1.0.0

package/dist/timescale.mjs

@@ -0,0 +1,471 @@
+// src/timescale.ts
+import { Pool } from "pg";
+var getPool = () => {
+  if (!global.__pgPool__) {
+    const host = process.env.PG_HOST || "127.0.0.1";
+    const port = Number(process.env.PG_PORT ?? 5432);
+    const user = process.env.PG_USER || "app";
+    const password = String(process.env.PG_PASSWORD ?? "app");
+    const database = process.env.PG_DATABASE || process.env.PG_DB || "app";
+    global.__pgPool__ = new Pool({
+      host,
+      port,
+      user,
+      password,
+      database,
+      max: 10,
+      idleTimeoutMillis: 3e4,
+      connectionTimeoutMillis: 5e3
+    });
+  }
+  return global.__pgPool__;
+};
+var derivativesSchemaReady = false;
+var spreadSchemaReady = false;
+var toRows = (symbol, interval, data) => data.map((i) => ({
+  symbol,
+  interval,
+  ts: new Date(i.timestamp),
+  // ms -> Date
+  open: i.open,
+  high: i.high,
+  low: i.low,
+  close: i.close,
+  volume: i.volume ?? null,
+  turnover: i.turnover ?? null
+}));
+async function upsertCandles(rows) {
+  if (!rows.length) return;
+  const pool = getPool();
+  const cols = [
+    "symbol",
+    "interval",
+    "ts",
+    "open",
+    "high",
+    "low",
+    "close",
+    "volume",
+    "turnover"
+  ];
+  const maxRows = Math.floor(65535 / cols.length);
+  if (rows.length > maxRows) {
+    for (let i = 0; i < rows.length; i += maxRows) {
+      await upsertCandles(rows.slice(i, i + maxRows));
+    }
+    return;
+  }
+  const valuesSql = rows.map(
+    (_, i) => `(${cols.map((__, j) => `$${i * cols.length + j + 1}`).join(",")})`
+  ).join(",");
+  const flat = rows.flatMap((r) => [
+    r.symbol,
+    r.interval,
+    r.ts,
+    r.open,
+    r.high,
+    r.low,
+    r.close,
+    r.volume ?? null,
+    r.turnover ?? null
+  ]);
+  const sql = `
+    INSERT INTO candles (${cols.join(",")})
+    VALUES ${valuesSql}
+    ON CONFLICT (symbol, interval, ts) DO UPDATE SET
+      open = EXCLUDED.open,
+      high = EXCLUDED.high,
+      low  = EXCLUDED.low,
+      close = EXCLUDED.close,
+      volume = COALESCE(EXCLUDED.volume, candles.volume),
+      turnover = COALESCE(EXCLUDED.turnover, candles.turnover)
+  `;
+  const client = await pool.connect();
+  try {
+    await client.query("BEGIN");
+    await client.query(sql, flat);
+    await client.query("COMMIT");
+  } catch (e) {
+    await client.query("ROLLBACK");
+    throw e;
+  } finally {
+    client.release();
+  }
+}
+var ensureDerivativesSchema = async () => {
+  if (derivativesSchemaReady) return;
+  const pool = getPool();
+  await pool.query("CREATE EXTENSION IF NOT EXISTS timescaledb");
+  await pool.query(`
+    CREATE TABLE IF NOT EXISTS derivatives_market (
+      symbol text NOT NULL,
+      interval text NOT NULL,
+      ts timestamptz NOT NULL,
+      open_interest double precision,
+      funding_rate double precision,
+      liq_long double precision,
+      liq_short double precision,
+      liq_total double precision,
+      source text,
+      ingested_at timestamptz NOT NULL DEFAULT now(),
+      PRIMARY KEY (symbol, interval, ts)
+    )
+  `);
+  await pool.query(`
+    SELECT create_hypertable(
+      'derivatives_market',
+      'ts',
+      if_not_exists => TRUE,
+      chunk_time_interval => interval '14 days'
+    )
+  `);
+  await pool.query(`
+    CREATE INDEX IF NOT EXISTS derivatives_market_symbol_tf_ts_idx
+    ON derivatives_market (symbol, interval, ts DESC)
+  `);
+  derivativesSchemaReady = true;
+};
+var ensureSpreadSchema = async () => {
+  if (spreadSchemaReady) return;
+  const pool = getPool();
+  await pool.query("CREATE EXTENSION IF NOT EXISTS timescaledb");
+  await pool.query(`
+    CREATE TABLE IF NOT EXISTS market_spread (
+      symbol text NOT NULL,
+      interval text NOT NULL,
+      ts timestamptz NOT NULL,
+      binance_price double precision,
+      coinbase_price double precision,
+      spread double precision,
+      source text,
+      ingested_at timestamptz NOT NULL DEFAULT now(),
+      PRIMARY KEY (symbol, interval, ts)
+    )
+  `);
+  await pool.query(`
+    SELECT create_hypertable(
+      'market_spread',
+      'ts',
+      if_not_exists => TRUE,
+      chunk_time_interval => interval '14 days'
+    )
+  `);
+  await pool.query(`
+    CREATE INDEX IF NOT EXISTS market_spread_symbol_tf_ts_idx
+    ON market_spread (symbol, interval, ts DESC)
+  `);
+  spreadSchemaReady = true;
+};
+async function upsertDerivatives(rows) {
+  if (!rows.length) return;
+  await ensureDerivativesSchema();
+  const pool = getPool();
+  const cols = [
+    "symbol",
+    "interval",
+    "ts",
+    "open_interest",
+    "funding_rate",
+    "liq_long",
+    "liq_short",
+    "liq_total",
+    "source"
+  ];
+  const maxRows = Math.floor(65535 / cols.length);
+  if (rows.length > maxRows) {
+    for (let i = 0; i < rows.length; i += maxRows) {
+      await upsertDerivatives(rows.slice(i, i + maxRows));
+    }
+    return;
+  }
+  const valuesSql = rows.map(
+    (_, i) => `(${cols.map((__, j) => `$${i * cols.length + j + 1}`).join(",")})`
+  ).join(",");
+  const flat = rows.flatMap((row) => [
+    row.symbol,
+    row.interval,
+    row.ts,
+    row.openInterest ?? null,
+    row.fundingRate ?? null,
+    row.liqLong ?? null,
+    row.liqShort ?? null,
+    row.liqTotal ?? null,
+    row.source ?? null
+  ]);
+  const sql = `
+    INSERT INTO derivatives_market (${cols.join(",")})
+    VALUES ${valuesSql}
+    ON CONFLICT (symbol, interval, ts) DO UPDATE SET
+      open_interest = COALESCE(EXCLUDED.open_interest, derivatives_market.open_interest),
+      funding_rate = COALESCE(EXCLUDED.funding_rate, derivatives_market.funding_rate),
+      liq_long = COALESCE(EXCLUDED.liq_long, derivatives_market.liq_long),
+      liq_short = COALESCE(EXCLUDED.liq_short, derivatives_market.liq_short),
+      liq_total = COALESCE(EXCLUDED.liq_total, derivatives_market.liq_total),
+      source = COALESCE(EXCLUDED.source, derivatives_market.source),
+      ingested_at = now()
+  `;
+  await pool.query(sql, flat);
+}
+async function getDerivativesRangeForSymbols(symbols, interval, startMs, endMs) {
+  if (!symbols.length)
+    return [];
+  await ensureDerivativesSchema();
+  const pool = getPool();
+  const sql = `
+    SELECT symbol, interval, ts, open_interest, funding_rate, liq_long, liq_short, liq_total
+    FROM derivatives_market
+    WHERE symbol = ANY($1)
+      AND interval = $2
+      AND ts >= to_timestamp($3/1000.0)
+      AND ts <= to_timestamp($4/1000.0)
+    ORDER BY symbol ASC, ts ASC
+  `;
+  const res = await pool.query(sql, [symbols, interval, startMs, endMs]);
+  return res.rows;
+}
+async function getDerivativesSummary(hours = 24, limit = 500) {
+  await ensureDerivativesSchema();
+  const pool = getPool();
+  const cappedHours = Math.max(1, Math.min(24 * 30, hours));
+  const cappedLimit = Math.max(50, Math.min(5e3, limit));
+  const rowsQ = await pool.query(
+    `
+      SELECT symbol, interval, ts, open_interest, funding_rate, liq_long, liq_short, liq_total
+      FROM derivatives_market
+      WHERE ts >= now() - ($1 || ' hours')::interval
+      ORDER BY ts DESC
+      LIMIT $2
+    `,
+    [String(cappedHours), cappedLimit]
+  );
+  const aggQ = await pool.query(
+    `
+      SELECT
+        symbol,
+        interval,
+        COUNT(*)::int AS points,
+        MAX(ts) AS last_ts,
+        AVG(open_interest) AS avg_open_interest,
+        AVG(funding_rate) AS avg_funding_rate,
+        SUM(COALESCE(liq_total, 0)) AS sum_liq_total
+      FROM derivatives_market
+      WHERE ts >= now() - ($1 || ' hours')::interval
+      GROUP BY symbol, interval
+      ORDER BY points DESC, symbol ASC
+      LIMIT 500
+    `,
+    [String(cappedHours)]
+  );
+  return {
+    rows: rowsQ.rows,
+    aggregates: aggQ.rows,
+    hours: cappedHours
+  };
+}
+async function upsertSpreadRows(rows) {
+  if (!rows.length) return;
+  await ensureSpreadSchema();
+  const pool = getPool();
+  const cols = [
+    "symbol",
+    "interval",
+    "ts",
+    "binance_price",
+    "coinbase_price",
+    "spread",
+    "source"
+  ];
+  const maxRows = Math.floor(65535 / cols.length);
+  if (rows.length > maxRows) {
+    for (let i = 0; i < rows.length; i += maxRows) {
+      await upsertSpreadRows(rows.slice(i, i + maxRows));
+    }
+    return;
+  }
+  const valuesSql = rows.map(
+    (_, i) => `(${cols.map((__, j) => `$${i * cols.length + j + 1}`).join(",")})`
+  ).join(",");
+  const flat = rows.flatMap((row) => [
+    row.symbol,
+    row.interval,
+    row.ts,
+    row.binancePrice ?? null,
+    row.coinbasePrice ?? null,
+    row.spread ?? null,
+    row.source ?? null
+  ]);
+  const sql = `
+    INSERT INTO market_spread (${cols.join(",")})
+    VALUES ${valuesSql}
+    ON CONFLICT (symbol, interval, ts) DO UPDATE SET
+      binance_price = COALESCE(EXCLUDED.binance_price, market_spread.binance_price),
+      coinbase_price = COALESCE(EXCLUDED.coinbase_price, market_spread.coinbase_price),
+      spread = COALESCE(EXCLUDED.spread, market_spread.spread),
+      source = COALESCE(EXCLUDED.source, market_spread.source),
+      ingested_at = now()
+  `;
+  await pool.query(sql, flat);
+}
+async function getSpreadRangeForSymbols(symbols, interval, startMs, endMs) {
+  if (!symbols.length) {
+    return [];
+  }
+  await ensureSpreadSchema();
+  const pool = getPool();
+  const sql = `
+    SELECT symbol, interval, ts, binance_price, coinbase_price, spread
+    FROM market_spread
+    WHERE symbol = ANY($1)
+      AND interval = $2
+      AND ts >= to_timestamp($3/1000.0)
+      AND ts <= to_timestamp($4/1000.0)
+    ORDER BY symbol ASC, ts ASC
+  `;
+  const res = await pool.query(sql, [symbols, interval, startMs, endMs]);
+  return res.rows;
+}
+async function getSpreadSummary(hours = 24, limit = 500) {
+  await ensureSpreadSchema();
+  const pool = getPool();
+  const cappedHours = Math.max(1, Math.min(24 * 30, hours));
+  const cappedLimit = Math.max(50, Math.min(5e3, limit));
+  const rowsQ = await pool.query(
+    `
+      SELECT symbol, interval, ts, binance_price, coinbase_price, spread
+      FROM market_spread
+      WHERE ts >= now() - ($1 || ' hours')::interval
+      ORDER BY ts DESC
+      LIMIT $2
+    `,
+    [String(cappedHours), cappedLimit]
+  );
+  const aggQ = await pool.query(
+    `
+      SELECT
+        symbol,
+        interval,
+        COUNT(*)::int AS points,
+        MAX(ts) AS last_ts,
+        AVG(spread) AS avg_spread,
+        STDDEV_POP(spread) AS std_spread
+      FROM market_spread
+      WHERE ts >= now() - ($1 || ' hours')::interval
+      GROUP BY symbol, interval
+      ORDER BY points DESC, symbol ASC
+      LIMIT 500
+    `,
+    [String(cappedHours)]
+  );
+  return {
+    rows: rowsQ.rows,
+    aggregates: aggQ.rows,
+    hours: cappedHours
+  };
+}
+async function getCandlesRange(symbol, interval, startMs, endMs) {
+  const pool = getPool();
+  const sql = `
+    SELECT symbol, interval, ts,
+           open, high, low, close, volume, turnover
+    FROM candles
+    WHERE symbol = $1 AND interval = $2
+      AND ts >= to_timestamp($3/1000.0)
+      AND ts <= to_timestamp($4/1000.0)
+    ORDER BY ts ASC
+  `;
+  const res = await pool.query(sql, [symbol, interval, startMs, endMs]);
+  return res.rows;
+}
+async function getDataEdges(symbol, interval) {
+  const pool = getPool();
+  const sqlMin = `
+    SELECT extract(epoch from ts)*1000 AS ms
+    FROM candles
+    WHERE symbol=$1 AND interval=$2
+    ORDER BY ts ASC
+    LIMIT 1
+  `;
+  const sqlMax = `
+    SELECT extract(epoch from ts)*1000 AS ms
+    FROM candles
+    WHERE symbol=$1 AND interval=$2
+    ORDER BY ts DESC
+    LIMIT 1
+  `;
+  const [minQ, maxQ] = await Promise.all([
+    pool.query(sqlMin, [symbol, interval]),
+    pool.query(sqlMax, [symbol, interval])
+  ]);
+  const minRaw = minQ.rows[0]?.ms;
+  const maxRaw = maxQ.rows[0]?.ms;
+  const min = Number.isFinite(Number(minRaw)) ? Number(minRaw) : void 0;
+  const max = Number.isFinite(Number(maxRaw)) ? Number(maxRaw) : void 0;
+  return { min, max };
+}
+async function waitForDbReady(attempts = 20, delayMs = 1e3) {
+  const pool = getPool();
+  let lastError;
+  for (let i = 0; i < attempts; i++) {
+    try {
+      await pool.query("SELECT 1");
+      return;
+    } catch (e) {
+      lastError = e;
+      await new Promise((r) => setTimeout(r, delayMs));
+    }
+  }
+  throw lastError;
+}
+async function deleteCandles(symbol, interval) {
+  const pool = getPool();
+  const sql = `
+    DELETE FROM candles
+    WHERE symbol = $1 AND interval = $2
+  `;
+  await pool.query(sql, [symbol, interval]);
+}
+async function findContinuityGap(symbol, interval) {
+  const pool = getPool();
+  const expectedSeconds = interval * 60;
+  const sql = `
+    WITH ordered AS (
+      SELECT
+        ts,
+        LAG(ts) OVER (ORDER BY ts) AS prev_ts
+      FROM candles
+      WHERE symbol = $1 AND interval = $2
+    )
+    SELECT
+      ts,
+      prev_ts,
+      EXTRACT(EPOCH FROM (ts - prev_ts))::int AS diff_seconds
+    FROM ordered
+    WHERE prev_ts IS NOT NULL
+      AND EXTRACT(EPOCH FROM (ts - prev_ts))::int <> $3
+    ORDER BY ts ASC
+    LIMIT 1
+  `;
+  const res = await pool.query(sql, [symbol, interval, expectedSeconds]);
+  const row = res.rows[0];
+  if (!row) return null;
+  return {
+    ts: new Date(row.ts).getTime(),
+    prevTs: new Date(row.prev_ts).getTime(),
+    diffSeconds: row.diff_seconds
+  };
+}
+export {
+  deleteCandles,
+  findContinuityGap,
+  getCandlesRange,
+  getDataEdges,
+  getDerivativesRangeForSymbols,
+  getDerivativesSummary,
+  getSpreadRangeForSymbols,
+  getSpreadSummary,
+  toRows,
+  upsertCandles,
+  upsertDerivatives,
+  upsertSpreadRows,
+  waitForDbReady
+};

package/package.json

@@ -0,0 +1,60 @@
+{
+  "name": "@tradejs/infra",
+  "version": "1.0.0",
+  "description": "Server-only infrastructure adapters for Redis, Timescale, ML, logging, and IO.",
+  "files": [
+    "dist",
+    "proto"
+  ],
+  "exports": {
+    "./files": {
+      "types": "./dist/files.d.ts",
+      "import": "./dist/files.mjs",
+      "require": "./dist/files.js"
+    },
+    "./http": {
+      "types": "./dist/http.d.ts",
+      "import": "./dist/http.mjs",
+      "require": "./dist/http.js"
+    },
+    "./logger": {
+      "types": "./dist/logger.d.ts",
+      "import": "./dist/logger.mjs",
+      "require": "./dist/logger.js"
+    },
+    "./ml": {
+      "types": "./dist/ml.d.ts",
+      "import": "./dist/ml.mjs",
+      "require": "./dist/ml.js"
+    },
+    "./redis": {
+      "types": "./dist/redis.d.ts",
+      "import": "./dist/redis.mjs",
+      "require": "./dist/redis.js"
+    },
+    "./timescale": {
+      "types": "./dist/timescale.d.ts",
+      "import": "./dist/timescale.mjs",
+      "require": "./dist/timescale.js"
+    }
+  },
+  "dependencies": {
+    "@grpc/grpc-js": "^1.10.7",
+    "@grpc/proto-loader": "^0.7.13",
+    "@tradejs/types": "^1.0.0",
+    "chalk": "4.1.2",
+    "ioredis": "5.8.0",
+    "pg": "8.16.3",
+    "winston": "3.17.0"
+  },
+  "devDependencies": {
+    "tsup": "^8.5.1",
+    "typescript": "^5.1"
+  },
+  "scripts": {
+    "build": "tsup",
+    "typecheck": "tsc -p ./tsconfig.json --noEmit"
+  },
+  "license": "MIT",
+  "author": "aleksnick (https://github.com/aleksnick)"
+}

package/proto/ml_infer.proto

@@ -0,0 +1,19 @@
+syntax = "proto3";
+
+package ml_infer;
+
+service MlInfer {
+  rpc Predict (PredictRequest) returns (PredictResponse) {}
+}
+
+message PredictRequest {
+  string strategy = 1;
+  map<string, double> features = 2;
+  double threshold = 3;
+}
+
+message PredictResponse {
+  double probability = 1;
+  double threshold = 2;
+  bool passed = 3;
+}