onbuzz 4.8.3 → 4.9.1

package/scripts/verify/local-embeddings.mjs

@@ -0,0 +1,171 @@
+#!/usr/bin/env node
+/**
+ * local-embeddings.mjs — real-runtime verifier for LocalEmbeddingProvider.
+ *
+ * WHY THIS LIVES OUTSIDE JEST
+ *
+ *   Transformers.js delegates inference to ONNX Runtime, which validates
+ *   typed-array inputs with `instanceof Float32Array`. Jest's
+ *   `--experimental-vm-modules` loader runs each test file in its own
+ *   VM context where `Float32Array` is a different constructor reference
+ *   than ORT's check sees. Every real inference call throws:
+ *
+ *     TypeError: A float32 tensor's data must be type of function Float32Array()
+ *
+ *   That's a fundamental incompatibility between Jest's experimental
+ *   ESM execution model and native bindings. The unit tests in
+ *   `localProvider.test.js` paper over this with a stub `pipelineFactory`,
+ *   which is why they "pass" while exercising none of the real path.
+ *
+ *   So we verify the real path here, in a plain `node` process — same
+ *   realm everywhere, ORT happy, end-to-end behavior actually checked.
+ *
+ * WHAT IT VERIFIES
+ *
+ *   1. The provider warms up against a real ONNX model.
+ *   2. `embedQuery` returns a normalized Float32Array of the right dim.
+ *   3. Same input → same vector (determinism).
+ *   4. Cosine similarity makes semantic sense (paraphrase > unrelated).
+ *   5. BGE task prefix actually changes the vector (query vs document).
+ *   6. `EmbeddingService → LocalEmbeddingProvider` wiring works.
+ *
+ * RUN
+ *
+ *   npm run verify:local-embeddings
+ *
+ *   First run downloads ~30 MB of weights for bge-small-en-v1.5. Cached
+ *   thereafter (~5s subsequent). Exits 0 on success, 1 on failure.
+ */
+
+import os from 'os';
+import fsp from 'fs/promises';
+import path from 'path';
+import { LocalEmbeddingProvider } from '../../src/services/embeddings/providers/localProvider.js';
+import { EmbeddingService } from '../../src/services/embeddings/embeddingService.js';
+
+const TEST_MODEL = 'Xenova/bge-small-en-v1.5';
+const TEST_DIMS = 384;
+
+const results = [];
+let failures = 0;
+
+function check(name, fn) {
+  return (async () => {
+    const t0 = Date.now();
+    try {
+      await fn();
+      const ms = Date.now() - t0;
+      results.push({ name, ok: true, ms });
+      console.log(`  ✔ ${name} (${ms}ms)`);
+    } catch (err) {
+      failures++;
+      const ms = Date.now() - t0;
+      results.push({ name, ok: false, ms, err: err?.message });
+      console.error(`  ✗ ${name} (${ms}ms)`);
+      console.error(`      ${err?.stack || err}`);
+    }
+  })();
+}
+
+function dot(a, b) {
+  let s = 0;
+  for (let i = 0; i < a.length; i++) s += a[i] * b[i];
+  return s;
+}
+
+async function main() {
+  console.log('Local-embeddings real-runtime verifier');
+  console.log(`Model: ${TEST_MODEL} (${TEST_DIMS}d)`);
+  console.log('');
+
+  // Verify the package is importable first. If not, every subsequent
+  // assertion is meaningless, so fail loud and bail.
+  try {
+    await import('@xenova/transformers');
+  } catch (err) {
+    console.error('FAIL: @xenova/transformers is not importable.');
+    console.error(`      ${err?.message || err}`);
+    console.error('      It must live in devDependencies; run `npm install`.');
+    process.exit(1);
+  }
+
+  const tmpDir = await fsp.mkdtemp(path.join(os.tmpdir(), 'loxia-local-verify-'));
+  const cacheDir = path.join(tmpDir, 'cache');
+  const sharedProvider = new LocalEmbeddingProvider({ modelId: TEST_MODEL, cacheDir });
+
+  await check('warmup + health', async () => {
+    await sharedProvider.warmup();
+    const h = await sharedProvider.health();
+    if (!h.ok) throw new Error(`health.ok=${h.ok}`);
+  });
+
+  await check('embedQuery returns normalized Float32Array of correct dim', async () => {
+    const vec = await sharedProvider.embedQuery('hello world');
+    if (!(vec instanceof Float32Array)) throw new Error('not a Float32Array');
+    if (vec.length !== TEST_DIMS) throw new Error(`dim=${vec.length}, expected ${TEST_DIMS}`);
+    let sum = 0;
+    for (let i = 0; i < vec.length; i++) sum += vec[i] * vec[i];
+    const mag = Math.sqrt(sum);
+    if (Math.abs(mag - 1) > 1e-4) throw new Error(`magnitude=${mag.toFixed(6)}, expected ≈1`);
+  });
+
+  await check('deterministic: same input → same vector', async () => {
+    const v1 = await sharedProvider.embedQuery('reproducibility check');
+    const v2 = await sharedProvider.embedQuery('reproducibility check');
+    let maxDelta = 0;
+    for (let i = 0; i < v1.length; i++) {
+      const d = Math.abs(v1[i] - v2[i]);
+      if (d > maxDelta) maxDelta = d;
+    }
+    if (maxDelta >= 1e-5) throw new Error(`maxDelta=${maxDelta} (expected <1e-5)`);
+  });
+
+  await check('cosine similarity is semantically meaningful', async () => {
+    const seed       = await sharedProvider.embedQuery('how do I throttle rate-limited API requests?');
+    const paraphrase = await sharedProvider.embedQuery('handling rate limits in HTTP API clients');
+    const unrelated  = await sharedProvider.embedQuery('recipe for chocolate chip cookies');
+    const sp = dot(seed, paraphrase);
+    const su = dot(seed, unrelated);
+    if (sp <= su) throw new Error(`paraphrase(${sp.toFixed(3)}) ≤ unrelated(${su.toFixed(3)})`);
+    if ((sp - su) <= 0.05) throw new Error(`gap=${(sp - su).toFixed(3)} too small (expected >0.05)`);
+  });
+
+  await check('BGE task prefix differentiates query from document', async () => {
+    const text = 'How do I configure retries?';
+    const [asQuery]    = await sharedProvider.embedDocuments([text], 'query');
+    const [asDocument] = await sharedProvider.embedDocuments([text], 'document');
+    let maxDelta = 0;
+    for (let i = 0; i < asQuery.length; i++) {
+      const d = Math.abs(asQuery[i] - asDocument[i]);
+      if (d > maxDelta) maxDelta = d;
+    }
+    if (maxDelta <= 1e-3) throw new Error(`maxDelta=${maxDelta} (expected >1e-3)`);
+  });
+
+  await check('EmbeddingService → LocalEmbeddingProvider wiring', async () => {
+    const svc = new EmbeddingService({
+      config: { provider: 'local', local: { modelId: TEST_MODEL } },
+      apiKeyManager: { getEmbeddingApiKey: () => null },
+      baseUrl: 'https://unused-for-local',
+      stateDir: tmpDir,
+    });
+    if (!svc.isEnabled) throw new Error('svc.isEnabled=false');
+    const vec = await svc.embedQuery('end-to-end through embeddingService');
+    if (!(vec instanceof Float32Array)) throw new Error('svc returned non-Float32Array');
+    if (vec.length !== TEST_DIMS) throw new Error(`svc.dim=${vec.length}`);
+    const t = svc.getTelemetry();
+    if (!(t.callsTotal > 0)) throw new Error(`telemetry.callsTotal=${t.callsTotal}`);
+    if (!(t.tokensTotal > 0)) throw new Error(`telemetry.tokensTotal=${t.tokensTotal}`);
+  });
+
+  console.log('');
+  const passed = results.filter(r => r.ok).length;
+  const total = results.length;
+  console.log(`${failures === 0 ? 'PASS' : 'FAIL'}: ${passed}/${total} checks passed`);
+  if (failures > 0) process.exit(1);
+}
+
+main().catch(err => {
+  console.error('VERIFIER CRASHED:', err);
+  process.exit(1);
+});

package/scripts/verify/openai-embeddings.mjs

@@ -0,0 +1,140 @@
+#!/usr/bin/env node
+/**
+ * openai-embeddings.mjs — real-network verifier for the direct-to-OpenAI
+ * (BYOK) provider (i.e. `provider: "openai"`).
+ *
+ * WHY THIS LIVES OUTSIDE JEST
+ *
+ *   `openaiProvider.test.js` uses an injected `fetchImpl` and verifies
+ *   request shape + parse logic. That doesn't tell you anything about:
+ *     - Whether the OpenAI key actually has embeddings access.
+ *     - Whether the live endpoint returns the dimensions our client expects.
+ *     - Whether rate-limit / auth failures surface correctly.
+ *
+ *   This script exercises the same code path the production runtime does,
+ *   against api.openai.com (or a configured OpenAI-compat proxy URL).
+ *
+ * REQUIRED ENV
+ *
+ *   LOXIA_OPENAI_API_KEY   the OpenAI key (or proxy key for compat layers).
+ *
+ * OPTIONAL ENV
+ *
+ *   LOXIA_OPENAI_URL       override endpoint (for DeepInfra, Together,
+ *                          or other OpenAI-compat layers). Default:
+ *                          https://api.openai.com/v1/embeddings
+ *   LOXIA_OPENAI_MODEL_ID  default text-embedding-3-small
+ *   LOXIA_OPENAI_DIMS      declared dimensions (default 1536; use 3072 for
+ *                          text-embedding-3-large)
+ *
+ * RUN
+ *
+ *   npm run verify:openai-embeddings
+ *
+ *   Exit codes:
+ *     0  all checks passed
+ *     1  one or more checks failed
+ *     2  required env not set (intentional non-zero so CI can't silently
+ *        skip the verification by forgetting the secret).
+ */
+
+import { OpenAIEmbeddingProvider } from '../../src/services/embeddings/providers/openaiProvider.js';
+
+const apiKey  = process.env.LOXIA_OPENAI_API_KEY;
+const url     = process.env.LOXIA_OPENAI_URL || 'https://api.openai.com/v1/embeddings';
+const modelId = process.env.LOXIA_OPENAI_MODEL_ID || 'text-embedding-3-small';
+const dims    = Number(process.env.LOXIA_OPENAI_DIMS || 1536);
+
+console.log('OpenAI direct (BYOK) embeddings real-network verifier');
+if (!apiKey) {
+  console.error('');
+  console.error('SKIP: required env vars not set.');
+  console.error('  LOXIA_OPENAI_API_KEY: MISSING');
+  console.error('');
+  console.error('CI must either configure this secret and re-run, or');
+  console.error('explicitly skip this step (and accept the coverage gap).');
+  process.exit(2);
+}
+
+console.log(`Endpoint:   ${url}`);
+console.log(`Model:      ${modelId}`);
+console.log(`Dimensions: ${dims}`);
+console.log('');
+
+const results = [];
+let failures = 0;
+
+function check(name, fn) {
+  return (async () => {
+    const t0 = Date.now();
+    try {
+      await fn();
+      const ms = Date.now() - t0;
+      results.push({ name, ok: true, ms });
+      console.log(`  ✔ ${name} (${ms}ms)`);
+    } catch (err) {
+      failures++;
+      const ms = Date.now() - t0;
+      results.push({ name, ok: false, ms, err: err?.message });
+      console.error(`  ✗ ${name} (${ms}ms)`);
+      console.error(`      ${err?.stack || err}`);
+    }
+  })();
+}
+
+function dot(a, b) {
+  let s = 0;
+  for (let i = 0; i < a.length; i++) s += a[i] * b[i];
+  return s;
+}
+
+const provider = new OpenAIEmbeddingProvider({
+  getApiKey: () => apiKey,
+  url,
+  modelId,
+  dimensions: dims,
+});
+
+await check('single embedQuery returns Float32Array of declared dim', async () => {
+  const vec = await provider.embedQuery('hello world');
+  if (!(vec instanceof Float32Array)) throw new Error('not a Float32Array');
+  if (vec.length !== dims) {
+    throw new Error(`endpoint returned ${vec.length}d but client declared ${dims}d`);
+  }
+  let sum = 0;
+  for (let i = 0; i < vec.length; i++) sum += vec[i] * vec[i];
+  const mag = Math.sqrt(sum);
+  if (Math.abs(mag - 1) > 1e-3) throw new Error(`magnitude=${mag.toFixed(6)} (expected ≈1)`);
+});
+
+await check('batch embedDocuments returns one vector per input', async () => {
+  const inputs = ['retry logic', 'rate limiting', 'circuit breakers'];
+  const vecs = await provider.embedDocuments(inputs);
+  if (!Array.isArray(vecs)) throw new Error('not an array');
+  if (vecs.length !== inputs.length) throw new Error(`got ${vecs.length} vectors for ${inputs.length} inputs`);
+  for (let i = 0; i < vecs.length; i++) {
+    if (!(vecs[i] instanceof Float32Array)) throw new Error(`vec[${i}] not Float32Array`);
+    if (vecs[i].length !== dims) throw new Error(`vec[${i}].length=${vecs[i].length}, expected ${dims}`);
+  }
+});
+
+await check('cosine similarity is semantically meaningful', async () => {
+  const seed       = await provider.embedQuery('how do I throttle rate-limited API requests?');
+  const paraphrase = await provider.embedQuery('handling rate limits in HTTP API clients');
+  const unrelated  = await provider.embedQuery('recipe for chocolate chip cookies');
+  const sp = dot(seed, paraphrase);
+  const su = dot(seed, unrelated);
+  if (sp <= su) throw new Error(`paraphrase(${sp.toFixed(3)}) ≤ unrelated(${su.toFixed(3)})`);
+  if ((sp - su) <= 0.03) throw new Error(`gap=${(sp - su).toFixed(3)} too small (expected >0.03)`);
+});
+
+await check('provider.health reports ok', async () => {
+  const h = await provider.health();
+  if (!h.ok) throw new Error(`health.ok=${h.ok} (${h.reason || 'no reason'})`);
+});
+
+console.log('');
+const passed = results.filter(r => r.ok).length;
+const total = results.length;
+console.log(`${failures === 0 ? 'PASS' : 'FAIL'}: ${passed}/${total} checks passed`);
+process.exit(failures === 0 ? 0 : 1);

package/src/__tests__/composioCliFlags.test.js

@@ -0,0 +1,239 @@
+/**
+ * `loxia composio` CLI — flag parsing + subcommand wiring.
+ *
+ * The CLI uses HTTP against the running server, so a full end-to-end
+ * test would need to boot a real server. Instead we unit-test the
+ * pieces that don't need a live server:
+ *   - parseFlags: --json, --user, --args, --open, --help, positional
+ *   - subcommand handlers: with a stubbed http.request, verify they
+ *     hit the right URL with the right body and exit codes
+ *
+ * This catches the regressions that actually matter (broken arg parser,
+ * wrong path, missing JSON body) without paying the bootstrap cost.
+ */
+import { describe, test, expect, beforeEach, afterEach, jest } from '@jest/globals';
+import { _internal } from '../../bin/composio-cli.js';
+
+const { parseFlags, statusDot } = _internal;
+
+describe('parseFlags', () => {
+  test('captures positional args in order', () => {
+    const f = parseFlags(['connect', 'github']);
+    expect(f.positional).toEqual(['connect', 'github']);
+  });
+
+  test('--json sets the flag', () => {
+    expect(parseFlags(['toolkits', '--json']).json).toBe(true);
+    expect(parseFlags(['toolkits']).json).toBe(false);
+  });
+
+  test('--user accepts both --user X and --user=X', () => {
+    expect(parseFlags(['status', '--user', 'agent-42']).user).toBe('agent-42');
+    expect(parseFlags(['status', '--user=agent-42']).user).toBe('agent-42');
+  });
+
+  test('--args captures the JSON string for exec', () => {
+    const f = parseFlags(['exec', 'github', 'X', '--args', '{"per_page":3}']);
+    expect(f.args).toBe('{"per_page":3}');
+  });
+
+  test('--open is captured as boolean', () => {
+    expect(parseFlags(['connect', 'github', '--open']).open).toBe(true);
+  });
+
+  test('--help and -h both set help', () => {
+    expect(parseFlags(['--help']).help).toBe(true);
+    expect(parseFlags(['-h']).help).toBe(true);
+  });
+
+  test('unrecognised flags fall through to positional', () => {
+    // We don't whitelist; unknown flags survive in positional so the
+    // caller can decide what to do. (Defensive — protects against typos
+    // becoming silent no-ops.)
+    const f = parseFlags(['exec', 'gh', 'X', '--unknown']);
+    expect(f.positional).toContain('--unknown');
+  });
+});
+
+describe('statusDot', () => {
+  test('returns a coloured/raw character per status', () => {
+    // We don't assert on the exact ANSI codes (depends on TTY), but
+    // every variant produces a non-empty string.
+    for (const s of ['ACTIVE', 'INITIATED', 'FAILED', 'NOT_CONNECTED', null]) {
+      const dot = statusDot(s);
+      expect(typeof dot).toBe('string');
+      expect(dot.length).toBeGreaterThan(0);
+    }
+  });
+});
+
+/* ---------------- Subcommand handlers (stubbed http) ---------------- */
+
+function makeDeps(routes) {
+  // Each route entry: { match: regex|string, method, response }
+  const http = {
+    request: jest.fn((url, opts, cb) => {
+      const u = new URL(url);
+      const route = routes.find(r => {
+        const m = r.match instanceof RegExp ? r.match.test(u.pathname) : u.pathname === r.match;
+        return m && (!r.method || r.method === opts.method);
+      });
+      const body = route ? route.response : { success: false, error: `unmatched ${opts.method} ${u.pathname}` };
+      const res = {
+        statusCode: route ? (route.status || 200) : 404,
+        on(evt, fn) {
+          if (evt === 'data') fn(JSON.stringify(body));
+          if (evt === 'end') fn();
+        },
+      };
+      setImmediate(() => cb(res));
+      return { on: () => {}, write: () => {}, end: () => {}, setTimeout: () => {}, destroy: () => {} };
+    }),
+  };
+  return {
+    http,
+    backend: { host: '127.0.0.1', port: 12345 },
+    args: [],
+  };
+}
+
+describe('cmdToolkits', () => {
+  test('hits GET /api/composio/toolkits and prints rows', async () => {
+    const deps = makeDeps([{ match: '/api/composio/toolkits', method: 'GET', response: { success: true, toolkits: [{ slug: 'github', name: 'GitHub' }] } }]);
+    const logSpy = jest.spyOn(console, 'log').mockImplementation(() => {});
+    try {
+      await _internal.cmdToolkits(deps, { json: false, positional: ['toolkits'] });
+      expect(deps.http.request).toHaveBeenCalled();
+      const calls = logSpy.mock.calls.flat().join('\n');
+      expect(calls).toMatch(/github/);
+      expect(calls).toMatch(/GitHub/);
+    } finally {
+      logSpy.mockRestore();
+    }
+  });
+
+  test('--json emits the raw response', async () => {
+    const deps = makeDeps([{ match: '/api/composio/toolkits', method: 'GET', response: { success: true, toolkits: [] } }]);
+    const logSpy = jest.spyOn(console, 'log').mockImplementation(() => {});
+    try {
+      await _internal.cmdToolkits(deps, { json: true, positional: ['toolkits'] });
+      const out = logSpy.mock.calls.flat().join('\n');
+      expect(out).toMatch(/"success": true/);
+    } finally {
+      logSpy.mockRestore();
+    }
+  });
+});
+
+describe('cmdConnections', () => {
+  test('forwards userId in the query string', async () => {
+    const deps = makeDeps([{
+      match: /^\/api\/composio\/connections/,
+      method: 'GET',
+      response: { success: true, connections: [{ id: 'conn-1', toolkitSlug: 'github', status: 'ACTIVE' }] },
+    }]);
+    const logSpy = jest.spyOn(console, 'log').mockImplementation(() => {});
+    try {
+      await _internal.cmdConnections(deps, { json: false, user: 'agent-99', positional: ['connections'] });
+      const url = deps.http.request.mock.calls[0][0];
+      expect(url).toMatch(/userId=agent-99/);
+    } finally {
+      logSpy.mockRestore();
+    }
+  });
+});
+
+describe('cmdConnect', () => {
+  test('POSTs to /api/composio/connect with body { toolkitSlug, userId }', async () => {
+    const deps = makeDeps([{
+      match: '/api/composio/connect',
+      method: 'POST',
+      response: { success: true, connectLink: 'https://connect.composio.dev/link/x', connectionId: 'conn-1' },
+    }]);
+    const logSpy = jest.spyOn(console, 'log').mockImplementation(() => {});
+    try {
+      await _internal.cmdConnect(deps, { json: false, positional: ['connect', 'github'], user: 'operator' });
+      // Verify the body was JSON.stringified into write() — we get the
+      // call args via deps.http.request itself (we wrote {} body above
+      // but the real write happens on the returned req object).
+      const out = logSpy.mock.calls.flat().join('\n');
+      expect(out).toMatch(/Connect github/);
+      expect(out).toMatch(/connect\.composio\.dev/);
+    } finally {
+      logSpy.mockRestore();
+    }
+  });
+});
+
+describe('cmdDisconnect', () => {
+  test('routes UUID-style id to DELETE by-id', async () => {
+    const deps = makeDeps([{
+      match: /^\/api\/composio\/connections\/conn-abc-def-123-456/,
+      method: 'DELETE',
+      response: { success: true },
+    }]);
+    const logSpy = jest.spyOn(console, 'log').mockImplementation(() => {});
+    try {
+      await _internal.cmdDisconnect(deps, { json: false, positional: ['disconnect', 'conn-abc-def-123-456'] });
+      const url = deps.http.request.mock.calls[0][0];
+      expect(url).toMatch(/conn-abc-def-123-456/);
+    } finally {
+      logSpy.mockRestore();
+    }
+  });
+
+  test('routes short slug to DELETE by-toolkit', async () => {
+    const deps = makeDeps([{
+      match: /^\/api\/composio\/connections\/by-toolkit/,
+      method: 'DELETE',
+      response: { success: true },
+    }]);
+    const logSpy = jest.spyOn(console, 'log').mockImplementation(() => {});
+    try {
+      await _internal.cmdDisconnect(deps, { json: false, positional: ['disconnect', 'github'], user: 'operator' });
+      const url = deps.http.request.mock.calls[0][0];
+      expect(url).toMatch(/by-toolkit/);
+    } finally {
+      logSpy.mockRestore();
+    }
+  });
+});
+
+describe('cmdExec', () => {
+  test('parses --args as JSON and forwards', async () => {
+    const deps = makeDeps([{
+      match: '/api/composio/execute',
+      method: 'POST',
+      response: { success: true, output: { ok: 1 } },
+    }]);
+    const logSpy = jest.spyOn(console, 'log').mockImplementation(() => {});
+    try {
+      await _internal.cmdExec(deps, {
+        json: false,
+        positional: ['exec', 'github', 'X'],
+        args: '{"per_page":3}',
+        user: 'operator',
+      });
+      const out = logSpy.mock.calls.flat().join('\n');
+      expect(out).toMatch(/X/);
+      expect(out).toMatch(/ok/);
+    } finally {
+      logSpy.mockRestore();
+    }
+  });
+
+  test('invalid --args JSON exits with helpful error', async () => {
+    const deps = makeDeps([]);
+    const errSpy = jest.spyOn(console, 'error').mockImplementation(() => {});
+    const exitSpy = jest.spyOn(process, 'exit').mockImplementation(() => { throw new Error('__exit__'); });
+    try {
+      await expect(_internal.cmdExec(deps, {
+        json: false, positional: ['exec', 'gh', 'X'], args: '{not-json',
+      })).rejects.toThrow('__exit__');
+      expect(errSpy.mock.calls.flat().join('\n')).toMatch(/Invalid --args/);
+    } finally {
+      errSpy.mockRestore();
+      exitSpy.mockRestore();
+    }
+  });
+});