@juspay/neurolink 12.19.2 → 12.20.1

package/dist/cli/utils/nativeTelemetry.js

@@ -0,0 +1,322 @@
+import { createHash } from "node:crypto";
+import { execFile } from "node:child_process";
+import { promisify } from "node:util";
+import { lstat, mkdir, mkdtemp, readFile, rm, writeFile, } from "node:fs/promises";
+import { dirname, isAbsolute, join, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+import yaml from "js-yaml";
+const execute = promisify(execFile);
+const profilePath = fileURLToPath(new URL("../../../scripts/observability/otel-collector.proxy-native-durable.yaml", import.meta.url));
+const mib = 1024 * 1024;
+const stagedFiles = [
+    "collector.yaml",
+    "collector.env.json",
+    "proxy.env",
+    "doctor.env",
+];
+const pendingChecks = [
+    "Run native-validate with the intended collector executable",
+    "Review source collector customizations against the staged standard profile; custom processors, TLS and exporter settings are not automatically copied",
+    "Create private queue/compaction directories (0700) on a volume with the declared quota; queue payload bounds do not enforce a disk quota",
+    "Apply and verify backend stream retention policies; preparation does not call the backend",
+    "Merge proxy and doctor environment snippets into existing settings; supply doctor backend authorization privately",
+    "Operator must back up config/service/env, explicitly activate through their service manager, and roll back those backups if health, export, stream or persistence checks fail",
+    "Reconcile fresh request, body and backend records after activation; validation alone is not runtime proof",
+];
+function record(value) {
+    if (!value || typeof value !== "object" || Array.isArray(value)) {
+        throw new Error("Expected an object in the native collector configuration");
+    }
+    return value;
+}
+function text(value, field) {
+    if (typeof value !== "string" || !value || /[\r\n\0]/.test(value)) {
+        throw new Error(`Missing or invalid ${field}`);
+    }
+    const resolved = value.replace(/\$\{env:([A-Za-z_][A-Za-z0-9_]*)(?::-([^}]*))?\}/g, (_match, key, fallback) => {
+        const replacement = process.env[key] ?? fallback;
+        if (replacement === undefined) {
+            throw new Error(`Set the source collector environment variable ${key} before preparation`);
+        }
+        return replacement;
+    });
+    if (!resolved || /\$\{|[\r\n\0]/.test(resolved)) {
+        throw new Error(`Unsupported substitution in ${field}`);
+    }
+    return resolved;
+}
+function positiveInteger(value, name, max) {
+    if (!Number.isSafeInteger(value) || value <= 0 || value > max) {
+        throw new Error(`${name} must be a positive integer no greater than ${max}`);
+    }
+    return value;
+}
+function privatePath(value, name) {
+    if (!isAbsolute(value) || /[\r\n\0]/.test(value)) {
+        throw new Error(`${name} must be an absolute path`);
+    }
+    const path = resolve(value);
+    if (path === dirname(path)) {
+        throw new Error(`${name} must not be a filesystem root`);
+    }
+    return path;
+}
+function loopbackListen(value, name) {
+    const endpoint = text(value, name);
+    const match = endpoint.match(/^127\.0\.0\.1:(\d+)$/);
+    if (!match) {
+        throw new Error(`${name} must use an explicit 127.0.0.1 listener`);
+    }
+    positiveInteger(Number(match[1]), name, 65535);
+    return endpoint;
+}
+function digest(value) {
+    return createHash("sha256").update(value).digest("hex");
+}
+/** Produce new private files only. Existing config, services and queues are untouched. */
+export async function prepareNativeTelemetry(args) {
+    const output = privatePath(args.output, "output");
+    const queue = privatePath(args.queueDirectory, "queue-directory");
+    const compact = privatePath(args.compactionDirectory, "compaction-directory");
+    if ([queue, compact].some((path) => path === output ||
+        path.startsWith(`${output}/`) ||
+        output.startsWith(`${path}/`))) {
+        throw new Error("Staging and collector storage paths must not overlap");
+    }
+    if (queue === compact ||
+        queue.startsWith(`${compact}/`) ||
+        compact.startsWith(`${queue}/`)) {
+        throw new Error("Queue and compaction directories must be separate, non-nested paths");
+    }
+    const metadataMib = positiveInteger(args.metadataQueueMib, "metadata-queue-mib", 1024);
+    const bodyMib = positiveInteger(args.bodyQueueMib, "body-queue-mib", 4096);
+    const quotaMib = positiveInteger(args.diskQuotaMib, "disk-quota-mib", 1048576);
+    if (quotaMib < 2 * (3 * metadataMib + bodyMib)) {
+        throw new Error("disk-quota-mib must allow at least twice all four queue payload limits for storage and compaction overhead");
+    }
+    positiveInteger(args.metadataRetentionDays, "metadata-retention-days", 3650);
+    positiveInteger(args.bodyRetentionDays, "body-retention-days", 3650);
+    positiveInteger(args.bodyPort, "body-port", 65535);
+    const sourcePath = resolve(args.collectorConfig);
+    const source = await readFile(sourcePath, "utf8");
+    let config;
+    try {
+        config = record(yaml.load(source));
+    }
+    catch {
+        // YAML parser diagnostics can contain complete source lines with credentials.
+        throw new Error("Could not parse the source collector YAML; source details are suppressed");
+    }
+    const exporter = record(record(config.exporters)["otlphttp/openobserve"]);
+    const headers = record(exporter.headers);
+    const endpoint = text(exporter.endpoint, "OpenObserve endpoint");
+    let url;
+    try {
+        url = new URL(endpoint);
+    }
+    catch {
+        throw new Error("Invalid OpenObserve endpoint");
+    }
+    if (url.username ||
+        url.password ||
+        url.search ||
+        url.hash ||
+        !/^\/api\/[a-zA-Z0-9_-]+\/?$/.test(url.pathname) ||
+        !(url.protocol === "https:" ||
+            (url.protocol === "http:" &&
+                ["127.0.0.1", "localhost", "[::1]"].includes(url.hostname)))) {
+        throw new Error("OpenObserve endpoint must be /api/<organization>, without embedded credentials; require HTTPS outside loopback");
+    }
+    const authorization = text(headers.Authorization ?? headers.authorization, "OpenObserve authorization");
+    const metadataStream = text(headers["stream-name"] ?? "neurolink_proxy", "metadata stream");
+    const bodyStream = args.bodyStream ?? `${metadataStream}_bodies`;
+    if (![metadataStream, bodyStream].every((value) => /^[a-zA-Z0-9_]+$/.test(value)) ||
+        metadataStream === bodyStream) {
+        throw new Error("Metadata and body streams must have distinct alphanumeric/underscore names");
+    }
+    const receivers = record(config.receivers);
+    const metadataListen = loopbackListen(record(record(record(receivers.otlp ?? receivers["otlp/metadata"]).protocols)
+        .http).endpoint, "metadata listener");
+    const extensions = record(config.extensions ?? {});
+    const healthListen = loopbackListen(record(extensions.health_check ?? { endpoint: "127.0.0.1:14333" }).endpoint, "health listener");
+    const telemetry = record(record(config.service).telemetry ?? {});
+    const metrics = record(telemetry.metrics ?? {});
+    const readers = Array.isArray(metrics.readers) ? metrics.readers : [];
+    const prometheus = readers
+        .map((reader) => record(record(record(reader).pull ?? {}).exporter ?? {}).prometheus)
+        .find((value) => value !== undefined);
+    const metricsPort = prometheus ? Number(record(prometheus).port) : 14388;
+    if (prometheus && record(prometheus).host !== "127.0.0.1") {
+        throw new Error("Collector metrics must use an explicit 127.0.0.1 listener");
+    }
+    positiveInteger(metricsPort, "metrics port", 65535);
+    const ports = [
+        Number(metadataListen.split(":")[1]),
+        Number(healthListen.split(":")[1]),
+        metricsPort,
+        args.bodyPort,
+    ];
+    if (new Set(ports).size !== ports.length) {
+        throw new Error("Metadata, bodies, health and metrics must use distinct ports");
+    }
+    const profile = await readFile(profilePath, "utf8");
+    const env = {
+        NEUROLINK_OTEL_METADATA_LISTEN: metadataListen,
+        NEUROLINK_OTEL_BODIES_LISTEN: `127.0.0.1:${args.bodyPort}`,
+        NEUROLINK_OTEL_HEALTH_LISTEN: healthListen,
+        NEUROLINK_OTEL_METRICS_PORT: String(metricsPort),
+        NEUROLINK_OTEL_QUEUE_DIRECTORY: queue,
+        NEUROLINK_OTEL_COMPACTION_DIRECTORY: compact,
+        NEUROLINK_OTEL_METADATA_QUEUE_BYTES: String(metadataMib * mib),
+        NEUROLINK_OTEL_BODY_QUEUE_BYTES: String(bodyMib * mib),
+        NEUROLINK_OPENOBSERVE_OTLP_ENDPOINT: endpoint,
+        NEUROLINK_OPENOBSERVE_BASIC_AUTH: authorization,
+        NEUROLINK_PROXY_STREAM_HEADER: metadataStream,
+        NEUROLINK_PROXY_BODY_STREAM_HEADER: bodyStream,
+    };
+    // Every value in these public snippets is independently restricted above.
+    const proxyEnv = `NEUROLINK_PROXY_LOG_SINK=otel\nOTEL_EXPORTER_OTLP_LOGS_ENDPOINT=http://${metadataListen}/v1/logs\nNEUROLINK_PROXY_OTLP_BODIES_ENDPOINT=http://127.0.0.1:${args.bodyPort}/v1/logs\n`;
+    const doctorEnv = `NEUROLINK_OPENOBSERVE_URL=${url.origin}\nNEUROLINK_OPENOBSERVE_ORG=${url.pathname.split("/")[2]}\nNEUROLINK_PROXY_STREAM_HEADER=${metadataStream}\nNEUROLINK_PROXY_BODY_STREAM_HEADER=${bodyStream}\nNEUROLINK_OTEL_COLLECTOR_METRICS_URL=http://127.0.0.1:${metricsPort}/metrics\n`;
+    const contents = [
+        profile,
+        `${JSON.stringify(env, null, 2)}\n`,
+        proxyEnv,
+        doctorEnv,
+    ];
+    const manifest = {
+        schemaVersion: 1,
+        activationImplemented: false,
+        source: { path: sourcePath, sha256: digest(source) },
+        files: Object.fromEntries(stagedFiles.map((name, index) => [name, digest(contents[index])])),
+        metadataStream,
+        bodyStream,
+        collectorMinimumVersion: "0.160.0",
+        queuePayloadBytes: {
+            metadataPerSignal: metadataMib * mib,
+            bodies: bodyMib * mib,
+            total: (3 * metadataMib + bodyMib) * mib,
+        },
+        operatorPolicy: {
+            diskQuotaBytes: quotaMib * mib,
+            metadataRetentionDays: args.metadataRetentionDays,
+            bodyRetentionDays: args.bodyRetentionDays,
+            enforced: false,
+        },
+        pending: pendingChecks,
+    };
+    // Exclusive creation refuses an existing destination, including a symlink.
+    await mkdir(output, { mode: 0o700 });
+    try {
+        for (let index = 0; index < stagedFiles.length; index++) {
+            await writeFile(join(output, stagedFiles[index]), contents[index], {
+                mode: 0o600,
+                flag: "wx",
+            });
+        }
+        await writeFile(join(output, "manifest.json"), `${JSON.stringify(manifest, null, 2)}\n`, { mode: 0o600, flag: "wx" });
+    }
+    catch {
+        await rm(output, { recursive: true, force: true });
+        throw new Error("Could not write the private staging directory; incomplete staged files were removed");
+    }
+    return {
+        directory: output,
+        prepared: true,
+        activated: false,
+        metadataStream,
+        bodyStream,
+        pending: pendingChecks,
+    };
+}
+/** Runs only the collector's static validator, with test-owned temporary storage. */
+export async function validateNativeTelemetry(args) {
+    const directory = privatePath(args.directory, "directory");
+    const binary = privatePath(args.collectorBin, "collector-bin");
+    const permissions = await lstat(directory);
+    if (!permissions.isDirectory() || (permissions.mode & 0o077) !== 0) {
+        throw new Error("Staging directory must be private (0700)");
+    }
+    for (const name of ["manifest.json", ...stagedFiles]) {
+        const permissions = await lstat(join(directory, name));
+        if (!permissions.isFile() || (permissions.mode & 0o077) !== 0) {
+            throw new Error("Staged files must be private regular files (0600)");
+        }
+    }
+    let manifest;
+    let env;
+    try {
+        manifest = record(JSON.parse(await readFile(join(directory, "manifest.json"), "utf8")));
+        env = record(JSON.parse(await readFile(join(directory, "collector.env.json"), "utf8")));
+    }
+    catch {
+        throw new Error("Invalid native telemetry stage; prepare a new directory");
+    }
+    if (manifest.schemaVersion !== 1 ||
+        manifest.activationImplemented !== false) {
+        throw new Error("Unsupported native telemetry staging manifest");
+    }
+    for (const name of stagedFiles) {
+        const file = join(directory, name);
+        if (digest(await readFile(file, "utf8")) !== record(manifest.files)[name]) {
+            throw new Error("Staged files changed or are not private; prepare a new directory before validation");
+        }
+    }
+    const source = record(manifest.source);
+    if (typeof source.path !== "string" ||
+        digest(await readFile(source.path, "utf8")) !== source.sha256) {
+        throw new Error("Source collector configuration changed; prepare again to avoid validating a stale migration");
+    }
+    const collectorEnv = {};
+    for (const [key, value] of Object.entries(env)) {
+        if (typeof value !== "string" ||
+            !/^(NEUROLINK_OTEL_|NEUROLINK_OPENOBSERVE_|NEUROLINK_PROXY_)/.test(key)) {
+            throw new Error("Invalid staged collector environment");
+        }
+        collectorEnv[key] = value;
+    }
+    let version;
+    try {
+        const result = await execute(binary, ["--version"], {
+            timeout: 15000,
+            maxBuffer: 65536,
+        });
+        const match = `${result.stdout}\n${result.stderr}`.match(/\b(\d+)\.(\d+)\.(\d+)([-+][\w.-]+)?\b/);
+        if (!match ||
+            match[4] ||
+            (Number(match[1]) === 0 && Number(match[2]) < 160)) {
+            throw new Error("unsupported");
+        }
+        version = `${match[1]}.${match[2]}.${match[3]}`;
+    }
+    catch {
+        throw new Error("Could not verify collector >= 0.160.0 (stable); executable diagnostics are suppressed to protect credentials");
+    }
+    const temporary = await mkdtemp(join(directory, "validation-"));
+    try {
+        await execute(binary, ["validate", "--config", join(directory, "collector.yaml")], {
+            env: {
+                ...process.env,
+                ...collectorEnv,
+                NEUROLINK_OTEL_QUEUE_DIRECTORY: join(temporary, "queue"),
+                NEUROLINK_OTEL_COMPACTION_DIRECTORY: join(temporary, "compact"),
+            },
+            timeout: 30000,
+            maxBuffer: 1048576,
+        });
+    }
+    catch {
+        throw new Error("Collector rejected the staged profile or validation timed out; executable diagnostics are suppressed to protect credentials");
+    }
+    finally {
+        await rm(temporary, { recursive: true, force: true });
+    }
+    return {
+        directory,
+        valid: true,
+        collectorVersion: version,
+        activated: false,
+        runtimeVerified: false,
+        pending: pendingChecks.slice(1),
+    };
+}
+//# sourceMappingURL=nativeTelemetry.js.map

package/dist/core/modules/MessageBuilder.js

@@ -104,6 +104,7 @@ export class MessageBuilder {
                     },
                     csvOptions: options.csvOptions,
                     pdfOptions: options.pdfOptions,
+                    imageOptions: options.imageOptions,
                     provider: options.provider,
                     model: options.model,
                     temperature: options.temperature,
@@ -226,6 +227,7 @@ export class MessageBuilder {
                     },
                     csvOptions: options.csvOptions,
                     pdfOptions: options.pdfOptions,
+                    imageOptions: options.imageOptions,
                     provider: options.provider,
                     model: options.model,
                     temperature: options.temperature,

package/dist/neurolink.js

@@ -4210,6 +4210,7 @@ Current user's request: ${currentInput}`;
             // reached convertToImages.
             csvOptions: options.csvOptions,
             pdfOptions: options.pdfOptions,
+            imageOptions: options.imageOptions,
             region: options.region,
             tts: options.tts,
             stt: options.stt,

package/dist/processors/config/fileExtensions.d.ts

@@ -13,7 +13,14 @@
  */
 /**
  * Image file extensions supported by the platform.
- * Includes `.svg`, which is processed as sanitized markup rather than raster.
+ *
+ * Includes `.svg`. On the processor path (`SvgProcessor`) that guarantees
+ * sanitized markup, never raster bytes — but that guarantee is scoped to
+ * that one consumer. This constant is also consumed by code that treats
+ * every image extension as raster (captioning, multimodal embedding, vision
+ * adapters); `.svg` is markup there too, just without the sanitization, so
+ * each such consumer must explicitly handle or reject `.svg` itself rather
+ * than assume this list implies raster-safety.
  */
 export declare const IMAGE_EXTENSIONS: readonly string[];
 /**

package/dist/processors/config/fileExtensions.js

@@ -17,7 +17,14 @@ import { extensionsForModality } from "./fileTypeRegistry.js";
 // =============================================================================
 /**
  * Image file extensions supported by the platform.
- * Includes `.svg`, which is processed as sanitized markup rather than raster.
+ *
+ * Includes `.svg`. On the processor path (`SvgProcessor`) that guarantees
+ * sanitized markup, never raster bytes — but that guarantee is scoped to
+ * that one consumer. This constant is also consumed by code that treats
+ * every image extension as raster (captioning, multimodal embedding, vision
+ * adapters); `.svg` is markup there too, just without the sanitization, so
+ * each such consumer must explicitly handle or reject `.svg` itself rather
+ * than assume this list implies raster-safety.
  */
 export const IMAGE_EXTENSIONS = extensionsForModality("image");
 /**

package/dist/proxy/bodyCapturePolicy.d.ts

@@ -0,0 +1,6 @@
+import type { ProxyBodyCaptureEntry, ProxyBodyCapturePolicyOptions, ProxyBodyCapturePolicySnapshot, ProxyBodyCapturePublication, ProxyBodyDeliveryResult, StoredBodyArtifact } from "../types/index.js";
+/** Diagnostic-only policy. Never receives or changes an inference body. */
+export declare function createProxyBodyCapturePolicy(options?: ProxyBodyCapturePolicyOptions): {
+    snapshot(): ProxyBodyCapturePolicySnapshot;
+    publish(entry: ProxyBodyCaptureEntry, stored: StoredBodyArtifact, publish: () => Promise<ProxyBodyDeliveryResult | undefined>): Promise<ProxyBodyCapturePublication>;
+};

package/dist/proxy/bodyCapturePolicy.js

@@ -0,0 +1,132 @@
+const MAX_REFERENCES = 1024;
+const REFERENCE_TTL_MS = 5 * 60_000;
+/** Diagnostic-only policy. Never receives or changes an inference body. */
+export function createProxyBodyCapturePolicy(options = {}) {
+    const now = options.now ?? (() => performance.now());
+    const configured = options.bytesPerMinute?.trim();
+    const budget = configured ? Number(configured) : null;
+    const invalidBudget = budget !== null && (!Number.isSafeInteger(budget) || budget <= 0);
+    const references = new Map();
+    let availableBytes = budget ?? Infinity;
+    let lastRefill = now();
+    let submittedBytes = 0;
+    let deduplicatedCaptures = 0;
+    let deduplicatedBytes = 0;
+    let policyExcludedCaptures = 0;
+    let policyExcludedBytes = 0;
+    const maintain = () => {
+        const at = now();
+        if (budget !== null && !invalidBudget) {
+            availableBytes = Math.min(budget, availableBytes + (Math.max(0, at - lastRefill) * budget) / 60_000);
+        }
+        lastRefill = at;
+        for (const [key, value] of references) {
+            if (value.expiresAt <= at) {
+                references.delete(key);
+            }
+        }
+    };
+    return {
+        snapshot() {
+            maintain();
+            return {
+                deduplicationEnabled: options.deduplicate !== false,
+                maxReferences: MAX_REFERENCES,
+                referenceTtlMs: REFERENCE_TTL_MS,
+                references: references.size,
+                byteBudgetPerMinute: invalidBudget ? null : budget,
+                invalidByteBudget: invalidBudget,
+                availableBytes: budget === null || invalidBudget ? null : Math.floor(availableBytes),
+                submittedBytes,
+                deduplicatedCaptures,
+                deduplicatedBytes,
+                policyExcludedCaptures,
+                policyExcludedBytes,
+            };
+        },
+        async publish(entry, stored, publish) {
+            maintain();
+            if (stored.redactedBody === undefined) {
+                return {};
+            }
+            const bytes = Buffer.byteLength(stored.redactedBody, "utf8");
+            const key = options.deduplicate !== false && stored.bodySha256 && entry.captureId
+                ? JSON.stringify([entry.requestId, stored.bodySha256, bytes])
+                : undefined;
+            const prior = key ? references.get(key) : undefined;
+            // Only detached diagnostic work waits. An uncertain export never becomes
+            // a reference, and references always point directly to an exported body.
+            if (prior &&
+                prior.reference.captureId !== entry.captureId &&
+                (await prior.acknowledged) &&
+                prior.expiresAt > now()) {
+                deduplicatedCaptures++;
+                deduplicatedBytes += bytes;
+                return {
+                    delivery: {
+                        status: "reference",
+                        reason: "identical_redacted_payload",
+                    },
+                    reference: { ...prior.reference },
+                };
+            }
+            maintain();
+            if (invalidBudget || bytes > availableBytes) {
+                policyExcludedCaptures++;
+                policyExcludedBytes += bytes;
+                return {
+                    delivery: {
+                        status: "policy_excluded",
+                        reason: invalidBudget
+                            ? "body_byte_budget_invalid"
+                            : "body_byte_budget_exhausted",
+                    },
+                };
+            }
+            availableBytes -= bytes;
+            submittedBytes += bytes;
+            let acknowledge = () => { };
+            const candidate = key && entry.captureId && stored.bodySha256
+                ? {
+                    reference: {
+                        captureId: entry.captureId,
+                        requestId: entry.requestId,
+                        bodySha256: stored.bodySha256,
+                        redactedBodyBytes: bytes,
+                        exportedAt: "",
+                    },
+                    expiresAt: now() + REFERENCE_TTL_MS,
+                    acknowledged: new Promise((resolve) => {
+                        acknowledge = resolve;
+                    }),
+                }
+                : undefined;
+            if (key && candidate) {
+                // Eviction only loses an optimization. It cannot invalidate an emitted
+                // reference, whose source remains an independently exported body.
+                if (references.size >= MAX_REFERENCES) {
+                    const oldest = references.keys().next().value;
+                    if (oldest !== undefined) {
+                        references.delete(oldest);
+                    }
+                }
+                references.set(key, candidate);
+            }
+            let accepted = false;
+            try {
+                const delivery = await publish();
+                accepted = delivery?.status === "transport_acknowledged";
+                if (candidate && accepted) {
+                    candidate.reference.exportedAt = new Date().toISOString();
+                }
+                return { delivery };
+            }
+            finally {
+                acknowledge(accepted);
+                if (key && !accepted && references.get(key) === candidate) {
+                    references.delete(key);
+                }
+            }
+        },
+    };
+}

package/dist/proxy/otelLogSink.d.ts

@@ -52,12 +52,22 @@ export declare function getProxyOtelLogSnapshot(): {
         exportUnconfirmed: number;
         dropped: number;
         outstanding: number;
+        outstandingBytes: number;
+        highWaterBytes: number;
+        byteLimitDrops: number;
+        retryingRecords: number;
+        retriedBatches: number;
+        retentionExpired: number;
+        lastTransientFailureAt: string | undefined;
         lastAcknowledgedAt: string | undefined;
         lastFailureAt: string | undefined;
         highWaterOutstanding: number;
         failureHistoryEvicted: number;
-        kind: "metadata" | "bodies";
+        kind: "metadata" | "bodies" | "indexes" | "diagnostics";
         capacity: number;
+        byteCapacity: number;
+        retentionMs: number;
+        oldestOutstandingAgeMs: number;
     }[];
 };
 /** Bounded provider flush belongs after final request and lifecycle publication. */