context-latch 1.0.1

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 pedro-arruda-moreira
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,41 @@
+# context-latch
+
+Smooth out LLM cold starts with intelligent request latching!
+
+A local proxy for developers running local LLMs on GPUs with limited VRAM. It batches incoming requests into a queue so the GPU never exceeds `num_parallel`, while keeping client connections alive so they don't time out waiting for a slot.
+
+## How it works
+
+When `num_parallel` is low (1–2), a GPU can only process a few requests at a time. Without context-latch, any extra requests arriving while the GPU is busy will fail or time out.
+
+context-latch sits between your client (e.g. Claude Desktop, OpenClaude CLI) and the local LLM server (llama.cpp, ollama, llama-swap):
+
+1. Incoming requests are queued immediately.
+2. A background scheduler ticks every `tickTime` ms and forwards queued requests only when a GPU slot opens.
+3. If a queued request exceeds `deadline` (and is still waiting), the proxy returns a **529** — clients like OpenClaude treat a 529 as a signal to retry, which resets the cold start.
+4. Browser requests bypass the queue entirely and always go straight to the GPU.
+
+## Configuration
+
+Copy `config_sample/config.json` to your project and adjust:
+
+| Field | Required | Description |
+|---|---|---|
+| `port` | yes | Port the proxy listens on |
+| `tickTime` | yes | Interval in ms to check for free GPU slots |
+| `deadline` | yes | Max wait time in ms before returning 529 |
+| `upstreamHost` | yes | Target LLM server host (e.g. `localhost:8080`) |
+| `upstreamType` | no | `ANTHROPIC` or `OPENAI` (default: `OPENAI`) |
+| `maxInflightRequests` | no | Parallel requests allowed on the GPU — matches `num_parallel` of the target LLM (default: no limit) |
+
+## Install
+
+```bash
+npm install -g context-latch
+```
+
+## Usage
+
+```bash
+context-latch ./path/to/config.json
+```

package/dist/bin/bin.d.ts

@@ -0,0 +1,3 @@
+#!/usr/bin/env node
+export {};
+//# sourceMappingURL=bin.d.ts.map

package/dist/bin/bin.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"bin.d.ts","sourceRoot":"","sources":["../../bin/bin.ts"],"names":[],"mappings":""}

package/dist/bin/bin.js

@@ -0,0 +1,25 @@
+#!/usr/bin/env node
+import { readFileSync } from "fs";
+import { mainInit } from "../index.js";
+import { UpstreamType } from "../core.js";
+const filePath = process.argv[2];
+const fileContent = filePath ? readFileSync(filePath, "utf-8") : null;
+if (!fileContent) {
+    console.error("No config file provided or file is empty.");
+    process.exit(1);
+}
+const config = JSON.parse(fileContent);
+if (config.upstreamType && typeof config.upstreamType === 'string') {
+    if (config.upstreamType.toLowerCase().indexOf('an') > -1) {
+        config.upstreamType = UpstreamType.ANTHROPIC;
+    }
+    else {
+        config.upstreamType = UpstreamType.OPENAI;
+    }
+}
+else {
+    config.upstreamType = UpstreamType.OPENAI;
+}
+const validatedConfig = config;
+mainInit(validatedConfig);
+//# sourceMappingURL=bin.js.map

package/dist/bin/bin.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"bin.js","sourceRoot":"","sources":["../../bin/bin.ts"],"names":[],"mappings":";AAEA,OAAO,EAAE,YAAY,EAAE,MAAM,IAAI,CAAC;AAClC,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AACvC,OAAO,EAAE,YAAY,EAAe,MAAM,YAAY,CAAC;AAGvD,MAAM,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AACjC,MAAM,WAAW,GAAG,QAAQ,CAAC,CAAC,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AACtE,IAAI,CAAC,WAAW,EAAE,CAAC;IACf,OAAO,CAAC,KAAK,CAAC,2CAA2C,CAAC,CAAC;IAC3D,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AACpB,CAAC;AAED,MAAM,MAAM,GAAwB,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;AAE5D,IAAI,MAAM,CAAC,YAAY,IAAI,OAAO,MAAM,CAAC,YAAY,KAAK,QAAQ,EAAE,CAAC;IACjE,IAAI,MAAM,CAAC,YAAY,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;QACvD,MAAM,CAAC,YAAY,GAAG,YAAY,CAAC,SAAS,CAAC;IACjD,CAAC;SAAM,CAAC;QACJ,MAAM,CAAC,YAAY,GAAG,YAAY,CAAC,MAAM,CAAC;IAC9C,CAAC;AACL,CAAC;KAAM,CAAC;IACJ,MAAM,CAAC,YAAY,GAAG,YAAY,CAAC,MAAM,CAAC;AAC9C,CAAC;AAED,MAAM,eAAe,GAAW,MAAgB,CAAC;AAEjD,QAAQ,CAAC,eAAe,CAAC,CAAC"}

package/dist/core.d.ts

@@ -0,0 +1,14 @@
+export declare enum UpstreamType {
+    ANTHROPIC = 0,
+    OPENAI = 1
+}
+export interface Config {
+    port: number;
+    tickTime: number;
+    deadline: number;
+    upstreamHost: string;
+    maxInflightRequests?: number;
+    upstreamType: UpstreamType;
+}
+export declare function doStart(config: Config): void;
+//# sourceMappingURL=core.d.ts.map

package/dist/core.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"core.d.ts","sourceRoot":"","sources":["../core.ts"],"names":[],"mappings":"AAMA,oBAAY,YAAY;IACpB,SAAS,IAAA;IACT,MAAM,IAAA;CACT;AAED,MAAM,WAAW,MAAM;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,YAAY,EAAE,YAAY,CAAC;CAC9B;AA8ED,wBAAgB,OAAO,CAAC,MAAM,EAAE,MAAM,QAoCrC"}

package/dist/core.js

@@ -0,0 +1,113 @@
+import { JobStatus } from "./job.js";
+import { enqueue, startQueueManager } from "./queue_manager.js";
+import { createServerInstance } from "./http_server.js";
+import { ERROR_STATUS_TIMED_OUT, httpRequest } from "./http_client.js";
+import { log } from "./log.js";
+export var UpstreamType;
+(function (UpstreamType) {
+    UpstreamType[UpstreamType["ANTHROPIC"] = 0] = "ANTHROPIC";
+    UpstreamType[UpstreamType["OPENAI"] = 1] = "OPENAI";
+})(UpstreamType || (UpstreamType = {}));
+let globalConfig;
+function sendJob(job) {
+    job.status = JobStatus.SENT;
+    log(job.id, 'Sending request to upstream host.');
+    const { request, response } = job;
+    response.on('close', () => {
+        if (job.status !== JobStatus.COMPLETED) {
+            log(job.id, 'Response closed before completion. Cancelling job.');
+            doCancelJob(job);
+        }
+    });
+    job.clientRequest = httpRequest(globalConfig.upstreamHost, request, (chunk) => {
+        if (job.isBrowser || globalConfig.upstreamType === UpstreamType.ANTHROPIC) {
+            if (job.status !== JobStatus.STREAMING) {
+                log(job.id, 'Streaming started.');
+            }
+            job.status = JobStatus.STREAMING;
+            response.write(chunk);
+            return;
+        }
+        if (chunk.toString().startsWith('data:')) {
+            if (job.status !== JobStatus.STREAMING) {
+                log(job.id, 'Streaming started.');
+                job.status = JobStatus.STREAMING;
+            }
+        }
+        if (job.status === JobStatus.STREAMING) {
+            response.write(chunk);
+        }
+    }, (error) => {
+        if (error) {
+            log(job.id, 'Error occurred while processing request.');
+            doCancelJob(job);
+        }
+        else {
+            response.end();
+            log(job.id, 'Request completed successfully.');
+            job.status = JobStatus.COMPLETED;
+        }
+    }, (statusCode, headers) => {
+        response.statusCode = statusCode;
+        for (const [key, value] of Object.entries(headers)) {
+            if (!value)
+                continue;
+            response.appendHeader(key, value);
+        }
+    });
+}
+function doCancelJob(job) {
+    job.status = JobStatus.ERROR;
+    const { request, response } = job;
+    job.clientRequest?.destroy(new Error(ERROR_STATUS_TIMED_OUT));
+    try {
+        response.statusCode = 529;
+        response.setHeader('Content-Type', 'text/plain');
+        response.write(`[context-latch] [Job:${job.id}] Request cancelled due to timeout. Failed by ${Date.now() - job.deadline} ms.`);
+    }
+    catch (e) {
+        log(job.id, 'Error while writing to response: ' + e);
+    }
+    response.end();
+    request.destroy(new Error(ERROR_STATUS_TIMED_OUT));
+    log(job.id, 'Request cancelled due to timeout.');
+}
+function isBrowserRequest(ua) {
+    if (!ua) {
+        return false;
+    }
+    if (typeof ua === 'string') {
+        return ua.indexOf('Mozilla') > -1;
+    }
+    else {
+        return isBrowserRequest(ua[0]);
+    }
+}
+export function doStart(config) {
+    globalConfig = config;
+    startQueueManager(config.maxInflightRequests, config.tickTime, sendJob, doCancelJob);
+    const server = createServerInstance((req, res) => {
+        const isBrowser = isBrowserRequest(req.headers['user-agent']);
+        let id = Math.random().toString(36).substring(2, 15);
+        if (isBrowser) {
+            id = `browser-${id}`;
+        }
+        const job = {
+            status: JobStatus.QUEUED,
+            request: req,
+            response: res,
+            deadline: Date.now() + config.deadline,
+            id: id,
+            isBrowser: isBrowser
+        };
+        res.on('destroy', () => {
+            job.status = JobStatus.ERROR;
+        });
+        log(job.id, 'Request received and queued.');
+        enqueue(job);
+    });
+    server.listen(config.port, 'localhost', () => {
+        console.log(`Server is running on http://localhost:${config.port}`);
+    });
+}
+//# sourceMappingURL=core.js.map

package/dist/core.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"core.js","sourceRoot":"","sources":["../core.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAY,MAAM,UAAU,CAAC;AAC/C,OAAO,EAAE,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAChE,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AACxD,OAAO,EAAE,sBAAsB,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AACvE,OAAO,EAAE,GAAG,EAAE,MAAM,UAAU,CAAC;AAE/B,MAAM,CAAN,IAAY,YAGX;AAHD,WAAY,YAAY;IACpB,yDAAS,CAAA;IACT,mDAAM,CAAA;AACV,CAAC,EAHW,YAAY,KAAZ,YAAY,QAGvB;AAWD,IAAI,YAAoB,CAAC;AAEzB,SAAS,OAAO,CAAC,GAAQ;IACrB,GAAG,CAAC,MAAM,GAAG,SAAS,CAAC,IAAI,CAAC;IAC5B,GAAG,CAAC,GAAG,CAAC,EAAE,EAAE,mCAAmC,CAAC,CAAC;IACjD,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,GAAG,CAAC;IAClC,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;QACtB,IAAI,GAAG,CAAC,MAAM,KAAK,SAAS,CAAC,SAAS,EAAE,CAAC;YACrC,GAAG,CAAC,GAAG,CAAC,EAAE,EAAE,oDAAoD,CAAC,CAAC;YAClE,WAAW,CAAC,GAAG,CAAC,CAAC;QACrB,CAAC;IACL,CAAC,CAAC,CAAC;IACH,GAAG,CAAC,aAAa,GAAG,WAAW,CAAC,YAAY,CAAC,YAAY,EAAE,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;QAC1E,IAAI,GAAG,CAAC,SAAS,IAAI,YAAY,CAAC,YAAY,KAAK,YAAY,CAAC,SAAS,EAAE,CAAC;YACxE,IAAI,GAAG,CAAC,MAAM,KAAK,SAAS,CAAC,SAAS,EAAE,CAAC;gBACrC,GAAG,CAAC,GAAG,CAAC,EAAE,EAAE,oBAAoB,CAAC,CAAC;YACtC,CAAC;YACD,GAAG,CAAC,MAAM,GAAG,SAAS,CAAC,SAAS,CAAC;YACjC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YACtB,OAAO;QACX,CAAC;QACD,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;YACvC,IAAI,GAAG,CAAC,MAAM,KAAK,SAAS,CAAC,SAAS,EAAE,CAAC;gBACrC,GAAG,CAAC,GAAG,CAAC,EAAE,EAAE,oBAAoB,CAAC,CAAC;gBAClC,GAAG,CAAC,MAAM,GAAG,SAAS,CAAC,SAAS,CAAC;YACrC,CAAC;QACL,CAAC;QACD,IAAI,GAAG,CAAC,MAAM,KAAK,SAAS,CAAC,SAAS,EAAE,CAAC;YACrC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAC1B,CAAC;IACL,CAAC,EAAE,CAAC,KAAK,EAAE,EAAE;QACT,IAAI,KAAK,EAAE,CAAC;YACR,GAAG,CAAC,GAAG,CAAC,EAAE,EAAE,0CAA0C,CAAC,CAAC;YACxD,WAAW,CAAC,GAAG,CAAC,CAAC;QACrB,CAAC;aAAM,CAAC;YACJ,QAAQ,CAAC,GAAG,EAAE,CAAC;YACf,GAAG,CAAC,GAAG,CAAC,EAAE,EAAE,iCAAiC,CAAC,CAAC;YAC/C,GAAG,CAAC,MAAM,GAAG,SAAS,CAAC,SAAS,CAAC;QACrC,CAAC;IACL,CAAC,EAAE,CAAC,UAAU,EAAE,OAAO,EAAE,EAAE;QACvB,QAAQ,CAAC,UAAU,GAAG,UAAU,CAAC;QACjC,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;YACjD,IAAI,CAAC,KAAK;gBAAE,SAAS;YACrB,QAAQ,CAAC,YAAY,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QACtC,CAAC;IACL,CAAC,CAAC,CAAC;AACP,CAAC;AAED,SAAS,WAAW,CAAC,GAAQ;IACzB,GAAG,CAAC,MAAM,GAAG,SAAS,CAAC,KAAK,CAAC;IAC7B,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,GAAG,CAAC;IAClC,GAAG,CAAC,aAAa,EAAE,OAAO,CAAC,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC,CAAC;IAC9D,IAAI,CAAC;QACD,QAAQ,CAAC,UAAU,GAAG,GAAG,CAAC;QAC1B,QAAQ,CAAC,SAAS,CAAC,cAAc,EAAE,YAAY,CAAC,CAAC;QACjD,QAAQ,CAAC,KAAK,CAAC,wBAAwB,GAAG,CAAC,EAAE,iDAAiD,IAAI,CAAC,GAAG,EAAE,GAAG,GAAG,CAAC,QAAQ,MAAM,CAAC,CAAC;IACnI,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACT,GAAG,CAAC,GAAG,CAAC,EAAE,EAAE,mCAAmC,GAAG,CAAC,CAAC,CAAC;IACzD,CAAC;IACD,QAAQ,CAAC,GAAG,EAAE,CAAC;IACf,OAAO,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC,CAAC;IACnD,GAAG,CAAC,GAAG,CAAC,EAAE,EAAE,mCAAmC,CAAC,CAAC;AACrD,CAAC;AAED,SAAS,gBAAgB,CAAC,EAAiC;IACvD,IAAI,CAAC,EAAE,EAAE,CAAC;QACN,OAAO,KAAK,CAAC;IACjB,CAAC;IACD,IAAI,OAAO,EAAE,KAAK,QAAQ,EAAE,CAAC;QACzB,OAAO,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC;IACtC,CAAC;SAAM,CAAC;QACJ,OAAO,gBAAgB,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IACnC,CAAC;AACL,CAAC;AAGD,MAAM,UAAU,OAAO,CAAC,MAAc;IAElC,YAAY,GAAG,MAAM,CAAC;IAEtB,iBAAiB,CACb,MAAM,CAAC,mBAAmB,EAC1B,MAAM,CAAC,QAAQ,EACf,OAAO,EACP,WAAW,CACd,CAAC;IAEF,MAAM,MAAM,GAAG,oBAAoB,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE;QAC7C,MAAM,SAAS,GAAG,gBAAgB,CAAC,GAAG,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC;QAC9D,IAAI,EAAE,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,SAAS,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACrD,IAAI,SAAS,EAAE,CAAC;YACZ,EAAE,GAAG,WAAW,EAAE,EAAE,CAAC;QACzB,CAAC;QACD,MAAM,GAAG,GAAQ;YACb,MAAM,EAAE,SAAS,CAAC,MAAM;YACxB,OAAO,EAAE,GAAG;YACZ,QAAQ,EAAE,GAAG;YACb,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC,QAAQ;YACtC,EAAE,EAAE,EAAE;YACN,SAAS,EAAE,SAAS;SACvB,CAAC;QACF,GAAG,CAAC,EAAE,CAAC,SAAS,EAAE,GAAG,EAAE;YACnB,GAAG,CAAC,MAAM,GAAG,SAAS,CAAC,KAAK,CAAC;QACjC,CAAC,CAAC,CAAC;QACH,GAAG,CAAC,GAAG,CAAC,EAAE,EAAE,8BAA8B,CAAC,CAAC;QAC5C,OAAO,CAAC,GAAG,CAAC,CAAC;IACjB,CAAC,CAAC,CAAC;IAGH,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,WAAW,EAAE,GAAG,EAAE;QACzC,OAAO,CAAC,GAAG,CAAC,yCAAyC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;IACxE,CAAC,CAAC,CAAC;AACP,CAAC"}

package/dist/http_client.d.ts

@@ -0,0 +1,4 @@
+import { ClientRequest, type IncomingMessage } from 'node:http';
+export declare const ERROR_STATUS_TIMED_OUT = "__TIMED_OUT__";
+export declare function httpRequest(upstreamHost: string, latchRequest: IncomingMessage, dataCallback: (chunk: any) => void, finishCallback: (error: Error | null) => void, statusCallback: (statusCode: number, headers: Record<string, string[] | undefined>) => void): ClientRequest;
+//# sourceMappingURL=http_client.d.ts.map

package/dist/http_client.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"http_client.d.ts","sourceRoot":"","sources":["../http_client.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAW,KAAK,eAAe,EAAE,MAAM,WAAW,CAAC;AAEzE,eAAO,MAAM,sBAAsB,kBAAkB,CAAC;AAUtD,wBAAgB,WAAW,CACvB,YAAY,EAAE,MAAM,EACpB,YAAY,EAAE,eAAe,EAC7B,YAAY,EAAE,CAAC,KAAK,EAAE,GAAG,KAAK,IAAI,EAClC,cAAc,EAAE,CAAC,KAAK,EAAE,KAAK,GAAG,IAAI,KAAK,IAAI,EAC7C,cAAc,EAAE,CAAC,UAAU,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,SAAS,CAAC,KAAK,IAAI,GAC5F,aAAa,CAyBf"}

package/dist/http_client.js

@@ -0,0 +1,64 @@
+import { ClientRequest, request } from 'node:http';
+export const ERROR_STATUS_TIMED_OUT = '__TIMED_OUT__';
+export function httpRequest(upstreamHost, latchRequest, dataCallback, finishCallback, statusCallback) {
+    const urlObj = new URL(latchRequest.url ?? '', `http://${upstreamHost}`);
+    const defaultHeaders = {
+        'Content-Type': latchRequest.headers['content-type'] || 'application/json',
+    };
+    const convertedHeaders = {};
+    for (const [key, value] of Object.entries(latchRequest.headers)) {
+        if (typeof value === 'string') {
+            convertedHeaders[key] = value;
+        }
+        else if (Array.isArray(value)) {
+            convertedHeaders[key] = value.join(',');
+        }
+    }
+    const options = {
+        hostname: urlObj.hostname,
+        port: urlObj.port ? parseInt(urlObj.port) : 80,
+        path: urlObj.pathname + urlObj.search,
+        method: latchRequest.method ?? 'GET',
+        headers: { ...defaultHeaders, ...convertedHeaders },
+    };
+    return performRequest(options, latchRequest, dataCallback, finishCallback, statusCallback);
+}
+function performRequest(options, latchRequest, dataCallback, finishCallback, statusCallback) {
+    function maybeFinishCallback(error) {
+        if (error.message !== ERROR_STATUS_TIMED_OUT) {
+            finishCallback(error);
+        }
+    }
+    const upstreamRequest = request(options, (upstreamResponse) => {
+        statusCallback(upstreamResponse.statusCode ?? 0, upstreamResponse.headersDistinct);
+        upstreamResponse.on('data', (chunk) => {
+            dataCallback(chunk);
+        });
+        upstreamResponse.on('end', () => {
+            finishCallback(null);
+        });
+        upstreamResponse.on('error', (error) => {
+            maybeFinishCallback(error);
+        });
+        upstreamResponse.on('aborted', (error) => {
+            maybeFinishCallback(error);
+        });
+    });
+    upstreamRequest.on('error', (error) => {
+        maybeFinishCallback(error);
+    });
+    latchRequest.on('data', (chunk) => {
+        upstreamRequest.write(chunk);
+    });
+    latchRequest.on('end', () => {
+        upstreamRequest.end();
+    });
+    latchRequest.on('error', (error) => {
+        maybeFinishCallback(error);
+    });
+    latchRequest.on('aborted', (error) => {
+        maybeFinishCallback(error);
+    });
+    return upstreamRequest;
+}
+//# sourceMappingURL=http_client.js.map

package/dist/http_client.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"http_client.js","sourceRoot":"","sources":["../http_client.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,OAAO,EAAwB,MAAM,WAAW,CAAC;AAEzE,MAAM,CAAC,MAAM,sBAAsB,GAAG,eAAe,CAAC;AAUtD,MAAM,UAAU,WAAW,CACvB,YAAoB,EACpB,YAA6B,EAC7B,YAAkC,EAClC,cAA6C,EAC7C,cAA2F;IAE3F,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,YAAY,CAAC,GAAG,IAAI,EAAE,EAAE,UAAU,YAAY,EAAE,CAAC,CAAC;IAEzE,MAAM,cAAc,GAA2B;QAC3C,cAAc,EAAE,YAAY,CAAC,OAAO,CAAC,cAAc,CAAC,IAAI,kBAAkB;KAC7E,CAAC;IAEF,MAAM,gBAAgB,GAA2B,EAAE,CAAC;IACpD,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC,OAAO,CAAC,EAAE,CAAC;QAC9D,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YAC5B,gBAAgB,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;QAClC,CAAC;aAAM,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;YAC9B,gBAAgB,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC5C,CAAC;IACL,CAAC;IAED,MAAM,OAAO,GAAmB;QAC5B,QAAQ,EAAE,MAAM,CAAC,QAAQ;QACzB,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE;QAC9C,IAAI,EAAE,MAAM,CAAC,QAAQ,GAAG,MAAM,CAAC,MAAM;QACrC,MAAM,EAAE,YAAY,CAAC,MAAM,IAAI,KAAK;QACpC,OAAO,EAAE,EAAE,GAAG,cAAc,EAAE,GAAG,gBAAgB,EAAE;KACtD,CAAC;IAEF,OAAO,cAAc,CAAC,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,cAAc,EAAE,cAAc,CAAC,CAAC;AAC/F,CAAC;AAED,SAAS,cAAc,CACnB,OAAuB,EACvB,YAA6B,EAC7B,YAAkC,EAClC,cAA6C,EAC7C,cAA2F;IAE3F,SAAS,mBAAmB,CAAC,KAAY;QACrC,IAAG,KAAK,CAAC,OAAO,KAAK,sBAAsB,EAAE,CAAC;YAC1C,cAAc,CAAC,KAAK,CAAC,CAAC;QAC1B,CAAC;IACL,CAAC;IACD,MAAM,eAAe,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,gBAAgB,EAAE,EAAE;QAE1D,cAAc,CAAC,gBAAgB,CAAC,UAAU,IAAI,CAAC,EAAE,gBAAgB,CAAC,eAAe,CAAC,CAAC;QAEnF,gBAAgB,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE;YAClC,YAAY,CAAC,KAAK,CAAC,CAAC;QACxB,CAAC,CAAC,CAAC;QAEH,gBAAgB,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;YAC5B,cAAc,CAAC,IAAI,CAAC,CAAC;QACzB,CAAC,CAAC,CAAC;QAEH,gBAAgB,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;YACnC,mBAAmB,CAAC,KAAK,CAAC,CAAC;QAC/B,CAAC,CAAC,CAAC;QAEH,gBAAgB,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,KAAK,EAAE,EAAE;YACrC,mBAAmB,CAAC,KAAK,CAAC,CAAC;QAC/B,CAAC,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;IAEH,eAAe,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;QAClC,mBAAmB,CAAC,KAAK,CAAC,CAAC;IAC/B,CAAC,CAAC,CAAC;IAEH,YAAY,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE;QAC9B,eAAe,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACjC,CAAC,CAAC,CAAC;IAEH,YAAY,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;QACxB,eAAe,CAAC,GAAG,EAAE,CAAC;IAC1B,CAAC,CAAC,CAAC;IAEH,YAAY,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;QAC/B,mBAAmB,CAAC,KAAK,CAAC,CAAC;IAC/B,CAAC,CAAC,CAAC;IAEH,YAAY,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,KAAK,EAAE,EAAE;QACjC,mBAAmB,CAAC,KAAK,CAAC,CAAC;IAC/B,CAAC,CAAC,CAAC;IAEH,OAAO,eAAe,CAAC;AAC3B,CAAC"}

package/dist/http_server.d.ts

@@ -0,0 +1,3 @@
+import { IncomingMessage, ServerResponse } from "node:http";
+export declare function createServerInstance(callback: (req: IncomingMessage, res: ServerResponse) => void): import("node:http").Server<typeof IncomingMessage, typeof ServerResponse>;
+//# sourceMappingURL=http_server.d.ts.map

package/dist/http_server.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"http_server.d.ts","sourceRoot":"","sources":["../http_server.ts"],"names":[],"mappings":"AAAA,OAAO,EAAgB,eAAe,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAE1E,wBAAgB,oBAAoB,CAAC,QAAQ,EAAE,CAAC,GAAG,EAAE,eAAe,EAAE,GAAG,EAAE,cAAc,KAAK,IAAI,6EAEjG"}

package/dist/http_server.js

@@ -0,0 +1,5 @@
+import { createServer, IncomingMessage, ServerResponse } from "node:http";
+export function createServerInstance(callback) {
+    return createServer(callback);
+}
+//# sourceMappingURL=http_server.js.map

package/dist/http_server.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"http_server.js","sourceRoot":"","sources":["../http_server.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAE1E,MAAM,UAAU,oBAAoB,CAAC,QAA6D;IAC9F,OAAO,YAAY,CAAC,QAAQ,CAAC,CAAC;AAClC,CAAC"}

package/dist/index.d.ts

@@ -0,0 +1,2 @@
+export { doStart as mainInit } from "./core.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,IAAI,QAAQ,EAAE,MAAM,WAAW,CAAC"}

package/dist/index.js

@@ -0,0 +1,2 @@
+export { doStart as mainInit } from "./core.js";
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,IAAI,QAAQ,EAAE,MAAM,WAAW,CAAC"}

package/dist/job.d.ts

@@ -0,0 +1,18 @@
+import type { ClientRequest, IncomingMessage, ServerResponse } from "node:http";
+export declare enum JobStatus {
+    QUEUED = 0,
+    SENT = 1,
+    STREAMING = 2,
+    COMPLETED = 3,
+    ERROR = 4
+}
+export interface Job {
+    status: JobStatus;
+    request: IncomingMessage;
+    response: ServerResponse;
+    deadline: number;
+    clientRequest?: ClientRequest;
+    id: string;
+    isBrowser: boolean;
+}
+//# sourceMappingURL=job.d.ts.map

package/dist/job.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"job.d.ts","sourceRoot":"","sources":["../job.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAEhF,oBAAY,SAAS;IACjB,MAAM,IAAA;IACN,IAAI,IAAA;IACJ,SAAS,IAAA;IACT,SAAS,IAAA;IACT,KAAK,IAAA;CACR;AAED,MAAM,WAAW,GAAG;IAChB,MAAM,EAAE,SAAS,CAAC;IAClB,OAAO,EAAE,eAAe,CAAC;IACzB,QAAQ,EAAE,cAAc,CAAC;IACzB,QAAQ,EAAE,MAAM,CAAC;IACjB,aAAa,CAAC,EAAE,aAAa,CAAC;IAC9B,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,OAAO,CAAC;CACtB"}

package/dist/job.js

@@ -0,0 +1,9 @@
+export var JobStatus;
+(function (JobStatus) {
+    JobStatus[JobStatus["QUEUED"] = 0] = "QUEUED";
+    JobStatus[JobStatus["SENT"] = 1] = "SENT";
+    JobStatus[JobStatus["STREAMING"] = 2] = "STREAMING";
+    JobStatus[JobStatus["COMPLETED"] = 3] = "COMPLETED";
+    JobStatus[JobStatus["ERROR"] = 4] = "ERROR";
+})(JobStatus || (JobStatus = {}));
+//# sourceMappingURL=job.js.map

package/dist/job.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"job.js","sourceRoot":"","sources":["../job.ts"],"names":[],"mappings":"AAEA,MAAM,CAAN,IAAY,SAMX;AAND,WAAY,SAAS;IACjB,6CAAM,CAAA;IACN,yCAAI,CAAA;IACJ,mDAAS,CAAA;IACT,mDAAS,CAAA;IACT,2CAAK,CAAA;AACT,CAAC,EANW,SAAS,KAAT,SAAS,QAMpB"}

package/dist/log.d.ts

@@ -0,0 +1,2 @@
+export declare function log(id: string, message: string): void;
+//# sourceMappingURL=log.d.ts.map

package/dist/log.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"log.d.ts","sourceRoot":"","sources":["../log.ts"],"names":[],"mappings":"AAAA,wBAAgB,GAAG,CAAC,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,QAE9C"}

package/dist/log.js

@@ -0,0 +1,4 @@
+export function log(id, message) {
+    console.log(`[${new Date().toLocaleTimeString()}] [${id}] ${message}`);
+}
+//# sourceMappingURL=log.js.map

package/dist/log.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"log.js","sourceRoot":"","sources":["../log.ts"],"names":[],"mappings":"AAAA,MAAM,UAAU,GAAG,CAAC,EAAU,EAAE,OAAe;IAC3C,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,IAAI,EAAE,CAAC,kBAAkB,EAAE,MAAM,EAAE,KAAK,OAAO,EAAE,CAAC,CAAC;AAC3E,CAAC"}

package/dist/queue_manager.d.ts

@@ -0,0 +1,4 @@
+import { type Job } from "./job.js";
+export declare function enqueue(job: Job): void;
+export declare function startQueueManager(maxInflightRequests: number | undefined, tickTime: number, jobSender: (job: Job) => void, jobCanceller: (job: Job) => void): void;
+//# sourceMappingURL=queue_manager.d.ts.map

package/dist/queue_manager.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"queue_manager.d.ts","sourceRoot":"","sources":["../queue_manager.ts"],"names":[],"mappings":"AAAA,OAAO,EAAa,KAAK,GAAG,EAAE,MAAM,UAAU,CAAC;AAM/C,wBAAgB,OAAO,CAAC,GAAG,EAAE,GAAG,QAE/B;AAaD,wBAAgB,iBAAiB,CAC7B,mBAAmB,EAAE,MAAM,GAAG,SAAS,EACvC,QAAQ,EAAE,MAAM,EAChB,SAAS,EAAE,CAAC,GAAG,EAAE,GAAG,KAAK,IAAI,EAC7B,YAAY,EAAE,CAAC,GAAG,EAAE,GAAG,KAAK,IAAI,QAuCnC"}

package/dist/queue_manager.js

@@ -0,0 +1,56 @@
+import { JobStatus } from "./job.js";
+import { log } from "./log.js";
+const queue = [];
+export function enqueue(job) {
+    queue.push(job);
+}
+setInterval(() => {
+    log('QueueManager', `Current queue length: ${queue.length}`);
+}, 25000);
+function shiftFromIndex(index) {
+    if (index < 0 || index >= queue.length) {
+        return;
+    }
+    queue.splice(index, 1);
+}
+export function startQueueManager(maxInflightRequests, tickTime, jobSender, jobCanceller) {
+    setInterval(() => {
+        if (queue.length <= 0) {
+            return;
+        }
+        const currentTime = Date.now();
+        let inflightRequests = 0;
+        for (let i = 0; i < queue.length;) {
+            const job = queue[i];
+            if (job.status >= JobStatus.COMPLETED) {
+                shiftFromIndex(i);
+                continue;
+            }
+            if (!job.isBrowser && currentTime > job.deadline && job.status < JobStatus.STREAMING) {
+                log(job.id, 'Job has exceeded its deadline. Cancelling job.');
+                jobCanceller(job);
+                shiftFromIndex(i);
+                continue;
+            }
+            if (job.status === JobStatus.STREAMING || job.status === JobStatus.SENT) {
+                i++;
+                if (!job.isBrowser) {
+                    inflightRequests++;
+                }
+                continue;
+            }
+            const aboveInflightLimit = maxInflightRequests !== undefined && inflightRequests >= maxInflightRequests;
+            if (aboveInflightLimit) {
+                if (job.isBrowser) {
+                    jobSender(job);
+                }
+            }
+            else {
+                jobSender(job);
+                inflightRequests++;
+            }
+            i++;
+        }
+    }, tickTime);
+}
+//# sourceMappingURL=queue_manager.js.map

package/dist/queue_manager.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"queue_manager.js","sourceRoot":"","sources":["../queue_manager.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAY,MAAM,UAAU,CAAC;AAC/C,OAAO,EAAE,GAAG,EAAE,MAAM,UAAU,CAAC;AAG/B,MAAM,KAAK,GAAU,EAAE,CAAA;AAEvB,MAAM,UAAU,OAAO,CAAC,GAAQ;IAC5B,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AACpB,CAAC;AAED,WAAW,CAAC,GAAG,EAAE;IACb,GAAG,CAAC,cAAc,EAAE,yBAAyB,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;AACjE,CAAC,EAAE,KAAK,CAAC,CAAC;AAEV,SAAS,cAAc,CAAC,KAAa;IACjC,IAAI,KAAK,GAAG,CAAC,IAAI,KAAK,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;QACrC,OAAO;IACX,CAAC;IACD,KAAK,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;AAC3B,CAAC;AAED,MAAM,UAAU,iBAAiB,CAC7B,mBAAuC,EACvC,QAAgB,EAChB,SAA6B,EAC7B,YAAgC;IAEhC,WAAW,CAAC,GAAG,EAAE;QACb,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YACpB,OAAO;QACX,CAAC;QACD,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC/B,IAAI,gBAAgB,GAAG,CAAC,CAAC;QACzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC;YAChC,MAAM,GAAG,GAAG,KAAK,CAAC,CAAC,CAAG,CAAC;YACvB,IAAI,GAAG,CAAC,MAAM,IAAI,SAAS,CAAC,SAAS,EAAE,CAAC;gBACpC,cAAc,CAAC,CAAC,CAAC,CAAC;gBAClB,SAAS;YACb,CAAC;YACD,IAAI,CAAC,GAAG,CAAC,SAAS,IAAI,WAAW,GAAG,GAAG,CAAC,QAAQ,IAAI,GAAG,CAAC,MAAM,GAAG,SAAS,CAAC,SAAS,EAAE,CAAC;gBACnF,GAAG,CAAC,GAAG,CAAC,EAAE,EAAE,gDAAgD,CAAC,CAAC;gBAC9D,YAAY,CAAC,GAAG,CAAC,CAAC;gBAClB,cAAc,CAAC,CAAC,CAAC,CAAC;gBAClB,SAAS;YACb,CAAC;YACD,IAAI,GAAG,CAAC,MAAM,KAAK,SAAS,CAAC,SAAS,IAAI,GAAG,CAAC,MAAM,KAAK,SAAS,CAAC,IAAI,EAAE,CAAC;gBACtE,CAAC,EAAE,CAAC;gBACJ,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,CAAC;oBACjB,gBAAgB,EAAE,CAAC;gBACvB,CAAC;gBACD,SAAS;YACb,CAAC;YACD,MAAM,kBAAkB,GAAG,mBAAmB,KAAK,SAAS,IAAI,gBAAgB,IAAI,mBAAmB,CAAC;YACxG,IAAI,kBAAkB,EAAE,CAAC;gBACrB,IAAI,GAAG,CAAC,SAAS,EAAE,CAAC;oBAChB,SAAS,CAAC,GAAG,CAAC,CAAC;gBACnB,CAAC;YACL,CAAC;iBAAM,CAAC;gBACJ,SAAS,CAAC,GAAG,CAAC,CAAC;gBACf,gBAAgB,EAAE,CAAC;YACvB,CAAC;YACD,CAAC,EAAE,CAAC;QACR,CAAC;IACL,CAAC,EAAE,QAAQ,CAAC,CAAC;AACjB,CAAC"}

package/package.json

@@ -0,0 +1,34 @@
+{
+  "name": "context-latch",
+  "version": "1.0.1",
+  "description": "Smooth out LLM cold starts with intelligent request latching!",
+  "homepage": "https://github.com/pedro-arruda-moreira/context-latch#readme",
+  "bugs": {
+    "url": "https://github.com/pedro-arruda-moreira/context-latch/issues"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/pedro-arruda-moreira/context-latch.git"
+  },
+  "license": "MIT",
+  "author": "",
+  "type": "module",
+  "main": "dist/index.js",
+  "bin": "dist/bin/bin.js",
+  "engines": {
+    "node": ">= 20"
+  },
+  "scripts": {
+    "build": "tsc",
+    "prepublishOnly": "npm run build"
+  },
+  "devDependencies": {
+    "@types/node": "^26.1.0",
+    "typescript": "^6.0.3"
+  },
+  "files": [
+    "dist/**/*.js",
+    "dist/**/*.d.ts",
+    "dist/**/*.map"
+  ]
+}