@netgreener/runtime 0.1.0

package/dist/uploader.js

@@ -0,0 +1,52 @@
+import { validateRuntimeSessionMetadata } from './contract/validate.js';
+export async function uploadRunSession(config, payload, fetchImpl = fetch) {
+    const validation = validateRuntimeSessionMetadata(payload.session_metadata);
+    if (!validation.ok) {
+        return {
+            ok: false,
+            error: `session_metadata contract failed: ${JSON.stringify(validation.issues)}`,
+        };
+    }
+    if (config.dryRun) {
+        return { ok: true, status: 0, body: payload, dryRun: true };
+    }
+    const url = `${config.apiUrl}/api/v1/runsessions/`;
+    const headers = {
+        Authorization: `Bearer ${config.token}`,
+        'Content-Type': 'application/json',
+        Accept: 'application/json',
+    };
+    if (config.organizationId != null) {
+        headers['X-Organization-ID'] = String(config.organizationId);
+    }
+    try {
+        const res = await fetchImpl(url, {
+            method: 'POST',
+            headers,
+            body: JSON.stringify(payload),
+        });
+        const text = await res.text();
+        let body = text;
+        try {
+            body = text ? JSON.parse(text) : null;
+        }
+        catch {
+            /* keep text */
+        }
+        if (!res.ok) {
+            return {
+                ok: false,
+                status: res.status,
+                error: `upload failed HTTP ${res.status}`,
+                body,
+            };
+        }
+        return { ok: true, status: res.status, body };
+    }
+    catch (err) {
+        return {
+            ok: false,
+            error: err instanceof Error ? err.message : String(err),
+        };
+    }
+}

package/dist/windowId.d.ts

@@ -0,0 +1,2 @@
+/** api_server pattern: rtw_ + 64 lowercase hex (68 chars total). */
+export declare function newRuntimeWindowId(): string;

package/dist/windowId.js

@@ -0,0 +1,7 @@
+import { createHash, randomBytes } from 'node:crypto';
+/** api_server pattern: rtw_ + 64 lowercase hex (68 chars total). */
+export function newRuntimeWindowId() {
+    const material = `${Date.now()}-${randomBytes(32).toString('hex')}`;
+    const digest = createHash('sha256').update(material).digest('hex');
+    return `rtw_${digest}`;
+}

package/examples/analyze-manifest.mjs

@@ -0,0 +1,30 @@
+/**
+ * MP4 Analyze deepen demo — disk scan + resource_finding_v1 candidates.
+ *
+ *   npm run example:analyze-manifest
+ *
+ * Still not CLI/VSCE wired; candidates only (no Optimize / verified savings).
+ */
+
+import { dirname, join } from 'node:path'
+import { fileURLToPath } from 'node:url'
+
+import {
+  buildResourceAnalyzeResultFromPaths,
+  isServiceProject,
+} from '../dist/analyze/index.js'
+
+const root = join(dirname(fileURLToPath(import.meta.url)), '../fixtures/analyze-sample')
+const result = buildResourceAnalyzeResultFromPaths(root, null, { projectId: '42' })
+
+console.log(JSON.stringify(result, null, 2))
+console.log(
+  isServiceProject(result.manifest)
+    ? 'ANALYZE MANIFEST PASS (service project)'
+    : 'ANALYZE MANIFEST: no service units detected',
+)
+console.log(
+  result.findings.length > 0
+    ? `ANALYZE FINDINGS PASS (${result.findings.length} resource_finding_v1 candidate(s))`
+    : 'ANALYZE FINDINGS: no candidates (unexpected for fixtures/analyze-sample)',
+)

package/examples/bullmq-live-smoke.mjs

@@ -0,0 +1,178 @@
+/**
+ * Real BullMQ + Redis live smoke (experimental N3).
+ *
+ * Requires:
+ *   - Redis (e.g. docker run -d --name netgreener-redis -p 6379:6379 redis:7-alpine)
+ *   - Local peers: `npm install bullmq ioredis --no-save`
+ *   - Live env (token / project / API); dry-run by default
+ *
+ *   $env:NETGREENER_RUNTIME_DRY_RUN="0"
+ *   $env:NETGREENER_API_URL="https://core-api.netgreener.com"
+ *   $env:NETGREENER_TOKEN="ngs_..."
+ *   $env:NETGREENER_PROJECT_ID="97"
+ *   node examples/bullmq-live-smoke.mjs
+ *
+ * Default export mode remains direct. Does not claim MP3 / managed sidecar.
+ */
+
+import { Queue, Worker } from 'bullmq'
+import IORedis from 'ioredis'
+
+import {
+  _resetOutboundInstrumentationForTests,
+  _resetRuntimeForTests,
+  getBullMqRuntime,
+  netgreenerBullMqProcessor,
+  validateExternalApiV0,
+  validateRuntimeSessionMetadata,
+} from '../dist/index.js'
+import {
+  extractPersistedRunId,
+  sessionMetadataForDogfood,
+} from '../dist/dogfood/tenantDogfoodGates.js'
+
+_resetRuntimeForTests()
+_resetOutboundInstrumentationForTests()
+
+const dryRun =
+  (process.env.NETGREENER_RUNTIME_DRY_RUN || '1').trim() !== '0' &&
+  (process.env.NETGREENER_RUNTIME_DRY_RUN || '1').trim().toLowerCase() !== 'false'
+const live = !dryRun
+
+process.env.NETGREENER_SERVICE_RUNTIME = '1'
+process.env.NETGREENER_TOKEN = process.env.NETGREENER_TOKEN || 'ngs_local_bullmq_smoke'
+process.env.NETGREENER_PROJECT_ID = process.env.NETGREENER_PROJECT_ID || '97'
+process.env.NETGREENER_RUNTIME_DRY_RUN = dryRun ? '1' : '0'
+process.env.NETGREENER_API_URL =
+  process.env.NETGREENER_API_URL || 'https://core-api.netgreener.com'
+process.env.NETGREENER_TENANT_SOURCE = process.env.NETGREENER_TENANT_SOURCE || 'task_kwarg'
+process.env.NETGREENER_TENANT_TASK_KWARG =
+  process.env.NETGREENER_TENANT_TASK_KWARG || 'organization_id'
+// Leave EXPORT_MODE unset → default direct
+delete process.env.NETGREENER_EXPORT_MODE
+
+const connection = new IORedis({
+  host: process.env.NETGREENER_REDIS_HOST || '127.0.0.1',
+  port: Number(process.env.NETGREENER_REDIS_PORT || '6379'),
+  maxRetriesPerRequest: null,
+})
+const queueName = process.env.NETGREENER_BULLMQ_QUEUE || 'netgreener-bullmq-smoke'
+
+console.log(
+  `mode: ${live ? 'LIVE upload' : 'DRY-RUN'}  redis=${connection.options.host}:${connection.options.port}  queue=${queueName}  project=${process.env.NETGREENER_PROJECT_ID}`,
+)
+
+const flushed = []
+const processor = netgreenerBullMqProcessor(
+  async (job) => {
+    // Light outbound so external_api_v0 can appear when metering is on.
+    try {
+      await fetch('https://api.openai.com/v1/models', {
+        method: 'GET',
+        headers: { Authorization: 'Bearer unused' },
+      })
+    } catch {
+      /* ignore network / auth noise — metering still records attempt if wrapped */
+    }
+    return { ok: true, organization_id: job.data?.organization_id }
+  },
+  {
+    onFlush: (result, payload) => {
+      flushed.push({ result, payload })
+      const mode = result.ok
+        ? result.dryRun
+          ? 'dry-run ok'
+          : `HTTP ${result.status}`
+        : `FAIL ${result.error}`
+      console.log(`[flush bullmq-real] ${mode}`)
+    },
+  },
+)
+
+const worker = new Worker(queueName, processor, { connection, concurrency: 1 })
+const queue = new Queue(queueName, { connection })
+
+const jobDone = new Promise((resolve, reject) => {
+  worker.on('completed', (job, result) => resolve({ job, result }))
+  worker.on('failed', (job, err) => reject(err))
+})
+
+try {
+  await queue.add(
+    'analyze_document',
+    { organization_id: 'org_bullmq_live', document_id: 'smoke-doc-1' },
+    { removeOnComplete: 10, removeOnFail: 10 },
+  )
+  const { job, result } = await jobDone
+  console.log(`[job] completed id=${job?.id} name=${job?.name} result=${JSON.stringify(result)}`)
+
+  const flushResult = await getBullMqRuntime().flush('manual')
+  if (!flushResult) {
+    throw new Error('flush returned null — no runtime/external samples recorded')
+  }
+
+  const flushPayload = flushed.at(-1)?.payload
+  const { sessionMetadata, uploadGate } = sessionMetadataForDogfood({
+    live,
+    flushResult,
+    flushPayload,
+    label: 'bullmq-real',
+  })
+  if (!uploadGate.ok) {
+    console.error(`[bullmq-real] ${uploadGate.reason}`)
+    process.exitCode = 1
+  } else {
+    const contract = validateRuntimeSessionMetadata(sessionMetadata)
+    if (!contract.ok) {
+      console.error('[bullmq-real] contract failed', contract.issues)
+      process.exitCode = 1
+    } else {
+      const ext = sessionMetadata.external_api_v0
+      if (ext) {
+        const extOk = validateExternalApiV0(ext)
+        if (!extOk.ok) console.warn('[bullmq-real] external_api_v0 issues', extOk.issues)
+      }
+      const runtime = sessionMetadata.service_runtime_v0
+      console.log(
+        `[bullmq-real] collector=${runtime?.collector} units=${(runtime?.units || [])
+          .map((u) => u.service_unit)
+          .join(',')}`,
+      )
+      console.log(
+        `[bullmq-real] by_tenant=${Object.keys(runtime?.by_tenant || {}).join(',') || '(none)'}`,
+      )
+      if (live) {
+        const runId = extractPersistedRunId(flushResult.body)
+        console.log(
+          `[bullmq-real] live upload ok status=${flushResult.status} run_id=${runId ?? 'n/a'} runtime_window_id=${flushPayload?.runtime_window_id ?? 'n/a'}`,
+        )
+      }
+      console.log(
+        live
+          ? 'BULLMQ LIVE SMOKE PASS (real Worker + Redis + upload)'
+          : 'BULLMQ DRY-RUN SMOKE PASS (real Worker + Redis)',
+      )
+    }
+  }
+} catch (err) {
+  console.error('BULLMQ SMOKE FAIL', err)
+  process.exitCode = 1
+} finally {
+  try {
+    await worker.close()
+  } catch {
+    /* ignore */
+  }
+  try {
+    await queue.close()
+  } catch {
+    /* ignore */
+  }
+  try {
+    await connection.quit()
+  } catch {
+    /* ignore */
+  }
+  _resetRuntimeForTests()
+  _resetOutboundInstrumentationForTests()
+}

package/examples/bullmq-mp3-worker-gate.mjs

@@ -0,0 +1,237 @@
+/**
+ * MP3 BullMQ worker gate — concurrency + restart (experimental).
+ *
+ * Requires Redis + optional peers:
+ *   docker run -d --name netgreener-redis -p 6379:6379 redis:7-alpine
+ *   npm install bullmq ioredis --no-save
+ *
+ * Dry-run (default — no api_server upload):
+ *   npm run example:bullmq-mp3-gate
+ *
+ * Live upload (authorized only):
+ *   $env:NETGREENER_RUNTIME_DRY_RUN="0"
+ *   $env:NETGREENER_API_URL="http://127.0.0.1:8000"
+ *   $env:NETGREENER_TOKEN="ngs_..."
+ *   $env:NETGREENER_PROJECT_ID="97"
+ *   npm run example:bullmq-mp3-gate
+ *
+ * Does NOT claim MP3 production support, managed sidecar, or npm publish.
+ * Default export mode remains direct.
+ */
+
+import { Queue, Worker } from 'bullmq'
+import IORedis from 'ioredis'
+
+import {
+  _resetOutboundInstrumentationForTests,
+  _resetRuntimeForTests,
+  getBullMqRuntime,
+  netgreenerBullMqProcessor,
+} from '../dist/index.js'
+import {
+  extractPersistedRunId,
+  sessionMetadataForDogfood,
+} from '../dist/dogfood/tenantDogfoodGates.js'
+import {
+  assertMp3ConcurrencyGate,
+  assertMp3RestartGate,
+  formatMp3GateFailure,
+} from '../dist/dogfood/mp3WorkerGates.js'
+
+_resetRuntimeForTests()
+_resetOutboundInstrumentationForTests()
+
+const dryRun =
+  (process.env.NETGREENER_RUNTIME_DRY_RUN || '1').trim() !== '0' &&
+  (process.env.NETGREENER_RUNTIME_DRY_RUN || '1').trim().toLowerCase() !== 'false'
+const live = !dryRun
+
+process.env.NETGREENER_SERVICE_RUNTIME = '1'
+process.env.NETGREENER_TOKEN = process.env.NETGREENER_TOKEN || 'ngs_local_bullmq_mp3'
+process.env.NETGREENER_PROJECT_ID = process.env.NETGREENER_PROJECT_ID || '97'
+process.env.NETGREENER_RUNTIME_DRY_RUN = dryRun ? '1' : '0'
+process.env.NETGREENER_API_URL =
+  process.env.NETGREENER_API_URL || 'http://127.0.0.1:8000'
+process.env.NETGREENER_TENANT_SOURCE = process.env.NETGREENER_TENANT_SOURCE || 'task_kwarg'
+process.env.NETGREENER_TENANT_TASK_KWARG =
+  process.env.NETGREENER_TENANT_TASK_KWARG || 'organization_id'
+delete process.env.NETGREENER_EXPORT_MODE
+
+const host = process.env.NETGREENER_REDIS_HOST || '127.0.0.1'
+const port = Number(process.env.NETGREENER_REDIS_PORT || '6379')
+const queueName = process.env.NETGREENER_BULLMQ_MP3_QUEUE || 'netgreener-bullmq-mp3-gate'
+const concurrency = Math.max(2, Number(process.env.NETGREENER_BULLMQ_CONCURRENCY || '2'))
+const jobCount = Math.max(concurrency * 2, Number(process.env.NETGREENER_BULLMQ_JOBS || '4'))
+const tenant = 'org_bullmq_mp3'
+const taskName = 'analyze_document'
+const taskUnit = `task:${taskName}`
+
+function makeProcessor(label) {
+  const flushed = []
+  const processor = netgreenerBullMqProcessor(
+    async (job) => {
+      await new Promise((r) => setTimeout(r, 30))
+      return { ok: true, organization_id: job.data?.organization_id, phase: label }
+    },
+    {
+      onFlush: (result, payload) => {
+        flushed.push({ result, payload })
+      },
+    },
+  )
+  return { processor, flushed }
+}
+
+async function waitJobs(worker, n, timeoutMs = 30000) {
+  let done = 0
+  return new Promise((resolve, reject) => {
+    const timer = setTimeout(() => {
+      reject(new Error(`timeout waiting for ${n} jobs (got ${done})`))
+    }, timeoutMs)
+    const onCompleted = () => {
+      done += 1
+      if (done >= n) {
+        clearTimeout(timer)
+        worker.off('completed', onCompleted)
+        worker.off('failed', onFailed)
+        resolve(done)
+      }
+    }
+    const onFailed = (_job, err) => {
+      clearTimeout(timer)
+      worker.off('completed', onCompleted)
+      worker.off('failed', onFailed)
+      reject(err)
+    }
+    worker.on('completed', onCompleted)
+    worker.on('failed', onFailed)
+  })
+}
+
+async function flushAndMeta(flushed, label) {
+  const flushResult = await getBullMqRuntime().flush('manual')
+  if (!flushResult) {
+    throw new Error(`${label}: flush returned null`)
+  }
+  const flushPayload = flushed.at(-1)?.payload
+  const { sessionMetadata, uploadGate } = sessionMetadataForDogfood({
+    live,
+    flushResult,
+    flushPayload,
+    label,
+  })
+  if (!uploadGate.ok) {
+    throw new Error(`${label}: ${uploadGate.reason}`)
+  }
+  const runId = live ? extractPersistedRunId(flushResult.body) : null
+  return { sessionMetadata, flushResult, runId, windowId: flushPayload?.runtime_window_id }
+}
+
+let connection
+let queue
+let exitCode = 0
+
+try {
+  connection = new IORedis({ host, port, maxRetriesPerRequest: null, lazyConnect: true })
+  await connection.connect()
+  await connection.ping()
+
+  console.log(
+    `MP3 worker gate  mode=${live ? 'LIVE' : 'DRY-RUN'}  redis=${host}:${port}  queue=${queueName}  concurrency=${concurrency}  jobs=${jobCount}`,
+  )
+
+  queue = new Queue(queueName, { connection })
+  await queue.obliterate({ force: true }).catch(() => {})
+
+  // --- Phase A: concurrency ---
+  _resetRuntimeForTests()
+  _resetOutboundInstrumentationForTests()
+  const a = makeProcessor('concurrency')
+  const workerA = new Worker(queueName, a.processor, { connection, concurrency })
+  for (let i = 0; i < jobCount; i += 1) {
+    await queue.add(
+      taskName,
+      { organization_id: tenant, document_id: `c-${i}` },
+      { removeOnComplete: 50, removeOnFail: 50 },
+    )
+  }
+  const completedA = await waitJobs(workerA, jobCount)
+  const metaA = await flushAndMeta(a.flushed, 'mp3-concurrency')
+  const gateA = assertMp3ConcurrencyGate({
+    phase: 'concurrency',
+    completedJobs: completedA,
+    minCompletedJobs: jobCount,
+    sessionMetadata: metaA.sessionMetadata,
+    expectedTenant: tenant,
+    expectedTaskUnit: taskUnit,
+  })
+  console.log(formatMp3GateFailure(gateA))
+  if (!gateA.ok) exitCode = 1
+  if (live && metaA.runId != null) {
+    console.log(`[concurrency] run_id=${metaA.runId} window=${metaA.windowId ?? 'n/a'}`)
+  }
+  await workerA.close()
+
+  // --- Phase B: restart (new Worker after close) ---
+  _resetRuntimeForTests()
+  _resetOutboundInstrumentationForTests()
+  const restartJobs = Math.max(2, concurrency)
+  const b = makeProcessor('restart')
+  const workerB = new Worker(queueName, b.processor, { connection, concurrency })
+  for (let i = 0; i < restartJobs; i += 1) {
+    await queue.add(
+      taskName,
+      { organization_id: tenant, document_id: `r-${i}` },
+      { removeOnComplete: 50, removeOnFail: 50 },
+    )
+  }
+  const completedB = await waitJobs(workerB, restartJobs)
+  const metaB = await flushAndMeta(b.flushed, 'mp3-restart')
+  const gateB = assertMp3RestartGate({
+    phase: 'restart',
+    completedJobs: completedB,
+    minCompletedJobs: restartJobs,
+    sessionMetadata: metaB.sessionMetadata,
+    expectedTenant: tenant,
+    expectedTaskUnit: taskUnit,
+  })
+  console.log(formatMp3GateFailure(gateB))
+  if (!gateB.ok) exitCode = 1
+  if (live && metaB.runId != null) {
+    console.log(`[restart] run_id=${metaB.runId} window=${metaB.windowId ?? 'n/a'}`)
+  }
+  await workerB.close()
+
+  if (exitCode === 0) {
+    console.log(
+      live
+        ? 'MP3 BULLMQ WORKER GATE PASS (concurrency + restart + live upload)'
+        : 'MP3 BULLMQ WORKER GATE PASS (concurrency + restart; dry-run)',
+    )
+    console.log('Not claimed: MP3 production support label, managed sidecar, npm publish')
+  }
+} catch (err) {
+  const msg = err && typeof err === 'object' && 'message' in err ? err.message : String(err)
+  if (/ECONNREFUSED|ENOTFOUND|connect/i.test(msg)) {
+    console.error(
+      'MP3 BULLMQ WORKER GATE BLOCKED: Redis not reachable. Start Redis (e.g. docker redis:7-alpine on :6379) and re-run. Unit tests for gate helpers still apply without Redis.',
+    )
+  } else {
+    console.error('MP3 BULLMQ WORKER GATE FAIL', err)
+  }
+  exitCode = 1
+} finally {
+  try {
+    if (queue) await queue.close()
+  } catch {
+    /* ignore */
+  }
+  try {
+    if (connection) await connection.quit()
+  } catch {
+    /* ignore */
+  }
+  _resetRuntimeForTests()
+  _resetOutboundInstrumentationForTests()
+  process.exitCode = exitCode
+}

package/examples/express-dry-run.mjs

@@ -0,0 +1,72 @@
+/**
+ * Local smoke without deploying api_server:
+ *
+ *   cd netgreener_node
+ *   npm run build
+ *   node examples/express-dry-run.mjs
+ *
+ * Prints one dry-run RunSession payload. This script always disables cloud upload,
+ * even when the caller's environment contains live credentials or DRY_RUN=false.
+ */
+
+import { createServer } from 'node:http'
+
+import {
+  getExpressRuntime,
+  loadRuntimeConfig,
+  netgreenerExpressMiddleware,
+} from '../dist/index.js'
+
+const config = loadRuntimeConfig({
+  ...process.env,
+  NETGREENER_SERVICE_RUNTIME: '1',
+  NETGREENER_TOKEN: process.env.NETGREENER_TOKEN || 'ngs_local_dry_run',
+  NETGREENER_PROJECT_ID: process.env.NETGREENER_PROJECT_ID || '1',
+  NETGREENER_RUNTIME_DRY_RUN: '1',
+})
+const mw = netgreenerExpressMiddleware({
+  config,
+  fetchImpl: async () => {
+    throw new Error('dry-run smoke attempted an outbound API request')
+  },
+  onFlush: (result) => {
+    console.log('flush:', result.ok ? (result.dryRun ? 'dry-run ok' : `HTTP ${result.status}`) : result.error)
+  },
+})
+
+const server = createServer((req, res) => {
+  const fakeReq = {
+    method: req.method,
+    path: (req.url || '/').split('?')[0],
+    url: req.url,
+  }
+  const fakeRes = {
+    statusCode: 200,
+    on(event, fn) {
+      if (event === 'finish' || event === 'close') {
+        res.on(event, fn)
+      }
+      return this
+    },
+  }
+  mw(fakeReq, fakeRes, () => {
+    res.statusCode = fakeReq.path === '/fail' ? 500 : 200
+    fakeRes.statusCode = res.statusCode
+    res.end(JSON.stringify({ path: fakeReq.path, status: res.statusCode }))
+  })
+})
+
+server.listen(0, async () => {
+  const { port } = server.address()
+  console.log(`dry-run server on http://127.0.0.1:${port}`)
+  await fetch(`http://127.0.0.1:${port}/health`)
+  await fetch(`http://127.0.0.1:${port}/v1/predict`, { method: 'POST' })
+  await fetch(`http://127.0.0.1:${port}/fail`)
+  const result = await getExpressRuntime().flush('manual')
+  if (!result?.ok || !result.dryRun) {
+    throw new Error('dry-run smoke did not return a dry-run result')
+  }
+  console.log(JSON.stringify(result?.ok ? result.body : result, null, 2))
+  server.close()
+  process.exit(0)
+})