@vike-io/cli 0.14.0 → 0.16.0

package/src/commands/stream.js

@@ -0,0 +1,308 @@
+import WebSocket from 'ws';
+import { requireApiKey } from '../auth.js';
+import { USER_AGENT, callTool, redact } from '../mcp-client.js';
+import { emit, emitError } from '../format.js';
+import { addTool } from './tool-command.js';
+
+// The four event streams on stream.vike.io; a channel belongs to one of them. The protocol (frames,
+// close codes, resume) is https://vike.io/docs/data/reference/stream-hl.
+export const STREAM_PATHS = ['hl', 'hl-cohorts', 'onchain', 'polymarket'];
+const MAX_SUBSCRIPTIONS = 100;          // per connection, the server's cap
+const IDLE_MS = 60_000;                 // the server pings every 15 s: a minute of silence is a dead connection
+const NUMBER_KEYS = new Set(['min_usd', 'min_amount', 'min_move']);
+const ONE_VALUE_KEYS = new Set(['side', 'direction', 'event', 'axis', 'move', 'role']);   // every other key is a list
+// No channel list here: the server decides which channels a path serves (`stream info` lists them) and refuses
+// an unknown one with `unknown_channel`.
+const FATAL_ERRORS = new Set(['bad_subscription', 'unknown_channel', 'rate_limited']);
+const FRESH_ERRORS = new Set(['cursor_expired', 'cursor_ahead']);      // resubscribe from now
+
+// `--filter k=v` pairs -> the subscribe frame's filter object. A list key takes comma-separated values and
+// repeats add to it; the server validates the keys and values and names a bad one.
+export function parseFilters(pairs = []) {
+  const filter = {};
+  for (const pair of pairs) {
+    const i = pair.indexOf('=');
+    if (i < 1) throw new Error(`--filter takes key=value, got ${JSON.stringify(pair)}`);
+    const key = pair.slice(0, i).trim();
+    const value = pair.slice(i + 1).trim();
+    if (NUMBER_KEYS.has(key)) {
+      const n = Number(value);
+      if (value === '' || !Number.isFinite(n)) throw new Error(`--filter ${key} must be a number, got ${JSON.stringify(value)}`);
+      filter[key] = n;
+    } else if (value === 'true' || value === 'false') {
+      filter[key] = value === 'true';       // a yes/no key, e.g. contract=false on chain.transfers
+    } else if (ONE_VALUE_KEYS.has(key)) {
+      filter[key] = value;
+    } else {
+      filter[key] = [...(filter[key] ?? []), ...value.split(',').map((v) => v.trim()).filter(Boolean)];
+    }
+  }
+  return filter;
+}
+
+// The later of two cursors; they are fixed-width, so text order is time order.
+const later = (a, b) => (b === undefined || (a !== undefined && a >= b) ? a : b);
+
+const backoff = (n) => Math.min(15_000, 1000 * 2 ** (n - 1));     // 1, 2, 4, 8, 15 s
+const MAX_CATCHUP_RETRIES = 5;      // then the subscription continues from now and says what it skipped
+
+// Run the subscriptions until a close the CLI must not retry; resolves to the exit code. Events and gap frames go
+// to `out` as received (one JSON object per line, so 64-bit integers stay exact); `out` returning false (stdout
+// full) pauses the socket until `drain` calls back. Status lines go to `log`. Every other end reconnects with the
+// last cursor of each subscription.
+export async function runStream({
+  url, apiKey, subs, out, log,
+  drain = (cb) => cb(),
+  WebSocketImpl = WebSocket,
+  sleep = (ms) => new Promise((r) => setTimeout(r, ms)),
+}) {
+  const cursors = new Map(subs.filter((s) => s.cursor).map((s) => [s.id, s.cursor]));
+  const ctx = { url, headers: { 'X-API-KEY': apiKey, 'User-Agent': USER_AGENT }, subs, cursors,
+    catchups: new Map(), out, drain, log, sleep, WS: WebSocketImpl };
+  let failures = 0;
+  for (;;) {
+    const r = await session(ctx);
+    if (r.exit !== undefined) return r.exit;
+    failures = r.opened ? 0 : failures + 1;
+    // A connection that had opened waits 1 s, so a server that keeps closing at once is not hammered. A handshake
+    // that keeps failing (stream restarting) retries at once, then after 1, 2, 4 ... 15 s.
+    const wait = r.opened ? 1000 : failures > 1 ? backoff(failures - 1) : 0;
+    if (wait) await sleep(wait);
+  }
+}
+
+function session({ url, headers, subs, cursors, catchups, out, drain, log, sleep, WS }) {
+  return new Promise((resolve) => {
+    let opened = false;
+    let done = false;
+    let paused = false;
+    let idle;
+    const ws = new WS(url, { headers, handshakeTimeout: 30_000 });
+    const finish = (r) => {
+      if (done) return;
+      done = true;
+      clearTimeout(idle);
+      resolve({ opened, ...r });
+    };
+    const stop = (exit) => { finish({ exit }); ws.terminate(); };
+    const alive = () => {
+      clearTimeout(idle);
+      if (paused) return;                 // no frames while we hold the socket: that silence is ours
+      idle = setTimeout(() => { log(`no frame for ${IDLE_MS / 1000} s; connecting again`); ws.terminate(); }, IDLE_MS);
+    };
+    // stdout is full (a slow reader): stop reading the socket instead of buffering without bound. Frames the socket
+    // had already read still arrive, and pings among them are answered first. The server closes a reader stalled
+    // for 30 s with 4408, and the reconnect resumes from the cursors.
+    const print = (text) => {
+      if (out(text) !== false || paused) return;
+      paused = true;
+      clearTimeout(idle);
+      ws.pause();
+      drain(() => { paused = false; if (!done) { ws.resume(); alive(); } });
+    };
+    const subscribe = (s) => {
+      const msg = { action: 'subscribe', id: s.id, channel: s.channel };
+      if (s.filter) msg.filter = s.filter;
+      if (cursors.has(s.id)) msg.cursor = cursors.get(s.id);
+      ws.send(JSON.stringify(msg));
+    };
+
+    ws.on('unexpected-response', (_req, res) => {
+      let body = '';
+      res.on('data', (chunk) => { body += chunk; });
+      res.on('end', () => {
+        const status = res.statusCode;
+        let why = body.slice(0, 200);
+        try { const j = JSON.parse(body); why = j.message ?? j.error?.message ?? why; } catch { /* HTML or empty */ }
+        // 401: the key; 403 / 410: the wrong address. Anything else (503, Cloudflare 521/522) is worth another try.
+        if ([401, 403, 410].includes(status)) {
+          log(`refused: HTTP ${status} ${why}`);
+          stop(1);
+        } else {
+          log(`refused: HTTP ${status} ${why}; connecting again`);
+          stop(undefined);
+        }
+      });
+    });
+    ws.on('error', (err) => { if (!done) log(`connection error: ${redact(err.message, headers['X-API-KEY'])}`); });
+    ws.on('open', () => {
+      opened = true;
+      alive();
+      for (const s of subs) subscribe(s);
+    });
+    ws.on('message', (data) => {
+      if (done) return;
+      alive();
+      const text = String(data);
+      let f;
+      try { f = JSON.parse(text); } catch { return; }
+      if (f.type === 'ping') {
+        ws.send('{"action":"pong"}');
+      } else if (f.type === 'event') {
+        for (const id of f.ids ?? []) {
+          cursors.set(id, later(cursors.get(id), f.cursor));
+          catchups.delete(id);            // the subscription is delivering: its catch-up retries start over
+        }
+        print(text);
+      } else if (f.type === 'gap') {
+        // About the channel, not one subscription: every subscription on it moves past the window.
+        for (const s of subs) if (s.channel === f.channel) cursors.set(s.id, later(cursors.get(s.id), f.cursor));
+        print(text);
+      } else if (f.type === 'subscribed') {
+        if (f.cursor) cursors.set(f.id, later(cursors.get(f.id), f.cursor));
+        log(`subscribed ${f.id} to ${f.channel} at ${f.cursor} (replay: ${f.replay?.from ?? 'none'})`);
+      } else if (f.type === 'error') {
+        log(`error: ${text}`);
+        const sub = subs.find((s) => s.id === f.id);
+        if (FATAL_ERRORS.has(f.code)) {
+          stop(1);
+        } else if (sub && FRESH_ERRORS.has(f.code)) {
+          cursors.delete(sub.id);
+          subscribe(sub);
+        } else if (sub && f.code === 'replay_too_large') {
+          if (!f.cursor || f.cursor === cursors.get(sub.id)) {
+            log('the replay is too large even from the last cursor: narrow the filter');
+            stop(1);
+            return;
+          }
+          cursors.set(sub.id, f.cursor);
+          subscribe(sub);
+        } else if (sub && f.code === 'catchup_unavailable') {
+          // The server could not read the replay (its store down or the read timing out): retry after 1, 2, 4,
+          // 8, 15 s, then continue from now and say which window was skipped.
+          if (f.cursor) cursors.set(sub.id, f.cursor);
+          const n = (catchups.get(sub.id) ?? 0) + 1;
+          if (n > MAX_CATCHUP_RETRIES) {
+            const from = cursors.get(sub.id);
+            catchups.delete(sub.id);
+            cursors.delete(sub.id);
+            log(`catch-up of ${sub.channel} failed ${MAX_CATCHUP_RETRIES} times: continuing from now. The events `
+              + `after cursor ${from} up to now were SKIPPED; read them with `
+              + `\`vike-data stream events ${sub.channel} --cursor ${from}\` (same --filter).`);
+            subscribe(sub);
+          } else {
+            catchups.set(sub.id, n);
+            sleep(backoff(n)).then(() => { if (!done) subscribe(sub); });
+          }
+        }
+        // slow_consumer, server_catching_up, replaced, memory_backstop, ping_timeout: the close follows.
+      }
+    });
+    ws.on('close', (code, reason) => {
+      if (done) return;
+      const why = String(reason ?? '');
+      if (code === 4410) {
+        log('replaced (4410): a newer connection of this key took over this address (one connection per key '
+          + 'per address, 3 per account). Not reconnecting: give each program its own key.');
+        finish({ exit: 1 });
+      } else if (code === 4401) {
+        log(`unauthorized (4401)${why ? `: ${why}` : ''}`);
+        finish({ exit: 1 });
+      } else {
+        log(`closed (${code}${why ? ` ${why}` : ''}); connecting again from the last cursors`);
+        finish({});
+      }
+    });
+  });
+}
+
+const FILTER_OPTION = ['--filter <key=value...>', 'Filter, repeatable: coin=BTC,ETH  min_usd=50000  wallet=0x...  side=buy'];
+
+export function registerStream(program) {
+  // `stream check --filter ...`: the subcommand's flags must stay the subcommand's, not go to `stream <path>`,
+  // which has the same names. Commander needs positional options on the parent chain for that; the root has
+  // only --version and --help, which keep working before a command name.
+  program.enablePositionalOptions();
+  const cmd = program
+    .command('stream <path>')
+    .enablePositionalOptions()
+    .description(`Print live events as NDJSON from wss://stream.vike.io/v1/stream/<path> (${STREAM_PATHS.join(', ')}); reconnects and resumes by itself`)
+    // Checked in the action, not requiredOption: commander checks a parent's required options for its subcommands too.
+    .option('--channel <channel...>', 'Required. Channel(s), one subscription each, e.g. hl.fills hl.liquidations (hl), chain.transfers (onchain), pm.trades (polymarket)')
+    .option(FILTER_OPTION[0], `${FILTER_OPTION[1]} (applies to every --channel)`)
+    .option('--cursor <cursor>', 'Resume from this cursor (from a frame, at most 3 days old) instead of starting now')
+    .action(async (path, opts) => {
+      try {
+        if (!STREAM_PATHS.includes(path)) {
+          throw new Error(`unknown stream ${JSON.stringify(path)}: use one of ${STREAM_PATHS.join(', ')}`);
+        }
+        if (!opts.channel) throw new Error('--channel is required, e.g. --channel hl.fills (`vike-data stream info` lists the channels)');
+        const channels = [...new Set(opts.channel)];
+        if (channels.length > MAX_SUBSCRIPTIONS) {
+          throw new Error(`at most ${MAX_SUBSCRIPTIONS} subscriptions per connection, got ${channels.length}`);
+        }
+        const filter = parseFilters(opts.filter);
+        const subs = channels.map((channel) => ({
+          id: channel.replace(/[^A-Za-z0-9_-]/g, '_').slice(0, 32),
+          channel,
+          filter: Object.keys(filter).length ? filter : undefined,
+          cursor: opts.cursor,
+        }));
+        const log = (line) => process.stderr.write(`vike-data stream: ${line}\n`);
+        // The reader went away (`| head -5`): that is the end the user asked for, not an error.
+        process.stdout.on('error', (err) => { if (err.code === 'EPIPE') process.exit(0); else throw err; });
+        log(`one connection per key on each address: another program streaming ${path} with this key replaces this one`);
+        process.exitCode = await runStream({
+          url: `wss://stream.vike.io/v1/stream/${path}`,
+          apiKey: requireApiKey(),
+          subs,
+          out: (line) => process.stdout.write(line + '\n'),
+          drain: (cb) => process.stdout.once('drain', cb),
+          log,
+        });
+      } catch (err) { emitError(err, opts); }
+    });
+
+  // The stream's MCP tools: no socket, so no connection of the key is replaced.
+  addTool(cmd, 'info', {
+    tool: 'stream_info',
+    desc: 'Every path, channel and filter key, the limits and close codes, from the server\'s own registry',
+  });
+
+  cmd
+    .command('check <channel>')
+    .description('Check a channel and filter with the stream server\'s own validator, without connecting (exit 1 if refused)')
+    .option(...FILTER_OPTION)
+    .option('--path <path>', `The endpoint the connection would use: ${STREAM_PATHS.join(' | ')} (default the channel's own)`)
+    .option('--json', 'Emit JSON')
+    .action(async (channel, opts) => {
+      try {
+        const filter = parseFilters(opts.filter);
+        const r = await callTool('stream_check_filter', {
+          channel, filter: Object.keys(filter).length ? filter : undefined, path: opts.path,
+        });
+        emit(r, opts);
+        if (r?.valid === false) process.exitCode = 1;
+      } catch (err) { emitError(err, opts); }
+    });
+
+  cmd
+    .command('events <channel>')
+    .description('Stored events of one channel after a cursor, oldest first (the REST catch-up, kept 3 days)')
+    .option(...FILTER_OPTION)
+    .option('--cursor <cursor>', 'next_cursor of the previous page, or a frame\'s cursor (omit for the oldest kept)')
+    .option('--since-ms <ms>', 'Start after this arrival time, epoch ms (instead of --cursor)')
+    .option('--limit <n>', 'Events per page, 1-1000 (default 200)')
+    .option('--json', 'Emit JSON')
+    .action(async (channel, opts) => {
+      try {
+        const filter = parseFilters(opts.filter);
+        emit(await callTool('stream_events', {
+          channel,
+          filter: Object.keys(filter).length ? filter : undefined,
+          cursor: opts.cursor,
+          since_ms: opts.sinceMs,
+          limit: opts.limit,
+        }), opts);
+      } catch (err) { emitError(err, opts); }
+    });
+
+  addTool(cmd, 'connections', {
+    tool: 'stream_connections',
+    desc: 'Your account\'s recent stream connections: path, key, state, close code (4410 = replaced), channels',
+    options: [
+      ['--hours <n>', 'How far back, 1-720 (default 168)'],
+      ['--limit <n>', 'Connections, newest first, 1-1000 (default 50)'],
+    ],
+  });
+}

package/src/commands/token.js

@@ -30,6 +30,7 @@ export function registerToken(program) {
     .option('--period <p>', '1h | 24h | 7d | 30d', '24h')
     .option('--tab <t>', 'bought | sold', 'sold')
     .option('--chain <chain>', 'ethereum | bsc | base | all', 'ethereum')
+    .option('--hide-contracts', 'Leave out pools, routers and other contracts (the website\'s "Hide pools & routers")')
     .option('--json', 'Emit JSON')
     .action(async (tokenAddress, opts) => {
       try {
@@ -38,6 +39,7 @@ export function registerToken(program) {
           period: opts.period,
           tab: opts.tab,
           chain: opts.chain,
+          hide_contracts: opts.hideContracts,
         });
         emit(r, opts);
       } catch (err) { emitError(err, opts); }

package/src/commands/tool-command.js

@@ -0,0 +1,27 @@
+import { callTool } from '../mcp-client.js';
+import { emit, emitError } from '../format.js';
+
+// Register `<group> <spec>` as one call to one MCP tool. `args` names the tool parameters the
+// positional arguments of `spec` fill, in order. Each flag is forwarded under its snake_case
+// parameter name (--from-ms -> from_ms) and an unset flag is not sent: a view-based tool refuses
+// a parameter its view does not take, and the server's defaults stay the defaults. Values go as
+// typed; the server casts them and names the fix when one is wrong.
+export function addTool(group, spec, { tool, desc, args = [], options = [] }) {
+  const cmd = group.command(spec).description(desc);
+  for (const [flags, text] of options) cmd.option(flags, text);
+  cmd.option('--json', 'Emit JSON');
+  cmd.action(async (...received) => {
+    received.pop(); // the Command
+    const opts = received.pop();
+    const payload = {};
+    args.forEach((name, i) => { payload[name] = received[i]; });
+    for (const [key, value] of Object.entries(opts)) {
+      if (value === undefined || key === 'json') continue;
+      payload[key.replace(/[A-Z]/g, (c) => `_${c.toLowerCase()}`)] = value;
+    }
+    try {
+      emit(await callTool(tool, payload), opts);
+    } catch (err) { emitError(err, opts); }
+  });
+  return cmd;
+}

package/src/commands/wallet.js

@@ -1,6 +1,7 @@
 import { callTool } from '../mcp-client.js';
 import { emit, emitError } from '../format.js';
 import { parseInt10 } from '../util.js';
+import { addTool } from './tool-command.js';
 
 export function registerWallet(program) {
   const cmd = program
@@ -184,4 +185,24 @@ export function registerWallet(program) {
         emit(r, opts);
       } catch (err) { emitError(err, opts); }
     });
+
+  const CHAIN = ['--chain <c>', 'ethereum | bsc | base | hyperevm (default ethereum)'];
+
+  addTool(cmd, 'first-funder <address>', {
+    tool: 'wallet_first_funder',
+    args: ['address'],
+    desc: 'The earliest inbound native coin and token transfers of a wallet, with the senders\' labels',
+    options: [
+      CHAIN,
+      ['--min-native <n>', 'Dust floor in WHOLE coins (default 0.0001; 0 = none)'],
+      ['--min-usd <n>', 'Floor for the token side in USD (default 1; 0 = unpriced included)'],
+    ],
+  });
+
+  addTool(cmd, 'exchange-usage <address>', {
+    tool: 'wallet_exchange_usage',
+    args: ['address'],
+    desc: 'Which centralised exchanges a wallet sent to and received from, with USD and shares',
+    options: [CHAIN, ['--window <w>', '30d | 90d | 180d | 365d | all (default 365d)']],
+  });
 }

package/src/index.js

@@ -13,7 +13,11 @@ import { registerInit } from './commands/init.js';
 import { registerInstall } from './commands/install.js';
 import { registerDoctor } from './commands/doctor.js';
 import { registerSchema } from './commands/schema.js';
-import { registerLogin, registerLogout } from './commands/auth.js';
+import { registerLogin, registerLogout, registerWhoami } from './commands/auth.js';
+import { registerCall } from './commands/call.js';
+import { registerHl } from './commands/hl.js';
+import { registerStream } from './commands/stream.js';
+import { registerSearch } from './commands/search.js';
 import { readPackageVersion } from './util.js';
 import {
   getUpdateNotification,
@@ -53,6 +57,7 @@ program.hook('postAction', (_thisCmd, actionCmd) => {
 
 registerLogin(program);
 registerLogout(program);
+registerWhoami(program);
 registerInit(program);
 registerInstall(program);
 registerDoctor(program);
@@ -64,6 +69,10 @@ registerAlerts(program);
 registerPolymarket(program);
 registerLabels(program);
 registerCandles(program);
+registerHl(program);
+registerStream(program);
+registerSearch(program);
+registerCall(program);
 
 const _parseT0 = Date.now();
 program.parseAsync(process.argv)

package/src/mcp-client.js

@@ -1,33 +1,49 @@
 import { requireApiKey, getEndpoint } from './auth.js';
+import { readPackageVersion } from './util.js';
+
+// The installed package's own version: npm_package_version is set only under `npm run`.
+export const USER_AGENT = `vike-cli/${readPackageVersion()}`;
 
 let _id = 0;
 function nextId() { return ++_id; }
 
+// A fetch error can quote a header value (an invalid one, e.g. a key pasted with a newline): never echo the key.
+export function redact(message, apiKey) {
+  return apiKey ? String(message).split(apiKey).join('vk_***') : String(message);
+}
+
+// fetch, with any error message cleared of the key.
+async function post(endpoint, apiKey, init) {
+  try {
+    return await fetch(endpoint, init);
+  } catch (err) {
+    throw Object.assign(new Error(redact(err.message, apiKey)), { name: err.name });
+  }
+}
+
 export async function callTool(toolName, args = {}, opts = {}) {
   const apiKey = opts.apiKey ?? requireApiKey();
   const endpoint = opts.endpoint ?? getEndpoint();
   const timeoutMs = opts.timeoutMs ?? 60_000;
 
-  const body = {
-    jsonrpc: '2.0',
-    id: nextId(),
-    method: 'tools/call',
-    params: { name: toolName, arguments: args },
-  };
+  // argsText: the caller's own JSON object text, sent byte for byte (a 77-digit number is not rounded).
+  const envelope = JSON.stringify({ jsonrpc: '2.0', id: nextId(), method: 'tools/call' });
+  const body = `${envelope.slice(0, -1)},"params":{"name":${JSON.stringify(toolName)},`
+    + `"arguments":${opts.argsText ?? JSON.stringify(args)}}}`;
 
   const ctrl = new AbortController();
   const t = setTimeout(() => ctrl.abort(), timeoutMs);
 
   let resp;
   try {
-    resp = await fetch(endpoint, {
+    resp = await post(endpoint, apiKey, {
       method: 'POST',
       headers: {
         'Content-Type': 'application/json',
         'X-API-KEY': apiKey,
-        'User-Agent': `vike-cli/${process.env.npm_package_version ?? 'dev'}`,
+        'User-Agent': USER_AGENT,
       },
-      body: JSON.stringify(body),
+      body,
       signal: ctrl.signal,
     });
   } catch (err) {
@@ -52,10 +68,15 @@ export async function callTool(toolName, args = {}, opts = {}) {
 
   // tools/call returns { content: [{type:'text', text: JSON}], isError }
   const content = json.result?.content?.[0]?.text;
-  if (!content) return json.result;
-  try { return JSON.parse(content); } catch {
-    return content;
+  let parsed = content;
+  try { parsed = JSON.parse(content); } catch { /* plain text */ }
+  if (json.result?.isError) {
+    // An in-band failure ({"error": ...}) is an error in every output mode, so the exit code is 1.
+    throw new Error(String(parsed?.error ?? parsed?.[0]?.error ?? content ?? 'tool error'));
   }
+  if (!content) return json.result;
+  // raw: the tool's text as sent, so integers above 2^53 survive.
+  return opts.raw ? content : parsed;
 }
 
 export async function listTools(opts = {}) {
@@ -67,9 +88,9 @@ export async function listTools(opts = {}) {
     method: 'tools/list',
     params: {},
   };
-  const resp = await fetch(endpoint, {
+  const resp = await post(endpoint, apiKey, {
     method: 'POST',
-    headers: { 'Content-Type': 'application/json', 'X-API-KEY': apiKey },
+    headers: { 'Content-Type': 'application/json', 'X-API-KEY': apiKey, 'User-Agent': USER_AGENT },
     body: JSON.stringify(body),
   });
   const json = await resp.json();
@@ -82,9 +103,9 @@ export async function ping(opts = {}) {
   // JSON-RPC "Invalid or missing API key" error (soon HTTP 401), so fail early.
   const endpoint = opts.endpoint ?? getEndpoint();
   const apiKey = opts.apiKey ?? requireApiKey();
-  const resp = await fetch(endpoint, {
+  const resp = await post(endpoint, apiKey, {
     method: 'POST',
-    headers: { 'Content-Type': 'application/json', 'X-API-KEY': apiKey },
+    headers: { 'Content-Type': 'application/json', 'X-API-KEY': apiKey, 'User-Agent': USER_AGENT },
     body: JSON.stringify({
       jsonrpc: '2.0', id: nextId(), method: 'initialize', params: {},
     }),