mycomesh-consumer 0.9.4 → 0.9.5

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mycomesh-consumer",
-  "version": "0.9.4",
+  "version": "0.9.5",
   "description": "MycoMesh V11 Consumer: a custodied deposit, relay-blind encrypted requests to any Provider, and a local OpenAI-compatible endpoint.",
   "repository": {
     "type": "git",

package/src/consumer.mjs

@@ -1,7 +1,7 @@
 // V11 Consumer runtime: one custodied deposit, any Provider, relay-blind requests.
 import { readFileSync } from "node:fs";
-import { createServer, request as httpRequest } from "node:http";
-import { request as httpsRequest } from "node:https";
+import { Agent as HttpAgent, createServer, request as httpRequest } from "node:http";
+import { Agent as HttpsAgent, request as httpsRequest } from "node:https";
 import { dirname, resolve } from "node:path";
 import { rootCertificates } from "node:tls";
 import { addressOf, encodeCall } from "./eip712.mjs";
@@ -28,10 +28,19 @@ export function loadNetwork(path) {
   return network;
 }
 
+/**
+ * A fresh connection per request, through an agent of its own: a reused keep-alive socket that the Relay's
+ * edge has just closed fails with ECONNRESET after the request may have been sent, which can only be
+ * reported as outcome unknown. (Only an agent's own createConnection is honoured, hence the pinned one here.)
+ */
 async function tlsOptions(target, ca, pin) {
-  if (target.protocol !== "https:") return {};
-  if (pin) return pinnedOptions(target.href, pin);
-  return ca ? { ca: [...rootCertificates, ca] } : {};
+  if (target.protocol !== "https:") return { agent: new HttpAgent({ keepAlive: false }) };
+  const agent = new HttpsAgent({ keepAlive: false });
+  if (pin) {
+    agent.createConnection = pinnedOptions(target.href, pin).createConnection;
+    return { agent };
+  }
+  return ca ? { agent, ca: [...rootCertificates, ca] } : { agent };
 }
 
 export async function httpJson(url, { method = "GET", body, ca, pin, timeoutMs = 330_000 } = {}) {
@@ -56,7 +65,10 @@ export async function httpJson(url, { method = "GET", body, ca, pin, timeoutMs =
         catch (error) { reject(error); }
       });
     });
-    req.setTimeout(timeoutMs, () => req.destroy(new Error("request timed out")));
+    // A wall-clock deadline, not a socket idle timeout: keep-alive sockets are reused, and an idle timeout
+    // left on one by an earlier short request (a 10 s /providers call) would later cut a long answer.
+    const timer = setTimeout(() => req.destroy(new Error(`request to ${target.host}${target.pathname} timed out after ${timeoutMs / 1000} s`)), timeoutMs);
+    req.on("close", () => clearTimeout(timer));
     req.on("error", reject);
     if (data) req.write(data);
     req.end();
@@ -96,7 +108,10 @@ export async function httpNdjson(url, { body, ca, pin, timeoutMs = 340_000, onLi
         try { resolvePromise({ status: res.statusCode, body: JSON.parse(chunks.join("") || "{}") }); } catch (error) { reject(error); }
       });
     });
-    req.setTimeout(timeoutMs, () => req.destroy(new Error("request timed out")));
+    // A wall-clock deadline, not a socket idle timeout: keep-alive sockets are reused, and an idle timeout
+    // left on one by an earlier short request (a 10 s /providers call) would later cut a long answer.
+    const timer = setTimeout(() => req.destroy(new Error(`request to ${target.host}${target.pathname} timed out after ${timeoutMs / 1000} s`)), timeoutMs);
+    req.on("close", () => clearTimeout(timer));
     req.on("error", reject);
     req.end(data);
   });
@@ -129,6 +144,7 @@ export class Consumer {
   constructor({ network, keyPrivate, maxFee, journalDir, fetchJson = httpJson, streamJson = httpNdjson, now = () => Math.floor(Date.now() / 1000) }) {
     this.streamJson = streamJson;
     this.journalDir = journalDir;
+    this.inFlight = new Map(); // Provider signer -> requests this Consumer has with it now
     this.network = network;
     this.keyPrivate = keyPrivate;
     this.key = addressOf(keyPrivate);
@@ -213,8 +229,24 @@ export class Consumer {
     return [...names].sort();
   }
 
-  /** Try Relays in order; fail over only while the request provably was not dispatched. */
-  async request({ endpoint, model, content, maxOutputTokens = 4096, options = {}, provider, onDelta }) {
+  /**
+   * Try Relays and Providers in order; fail over only while the request provably was not dispatched. When
+   * every Provider is busy, wait and try again (a short queue) rather than failing a burst of requests.
+   */
+  async request(args) {
+    const deadline = Date.now() + (this.queueMs ?? 120_000);
+    for (let attempt = 0; ; attempt += 1) {
+      try {
+        return await this.attempt(args);
+      } catch (error) {
+        const busy = error.dispatched === false && [502, 503].includes(error.status);
+        if (!busy || Date.now() > deadline) throw error;
+        await new Promise((resolve) => setTimeout(resolve, Math.min(4_000, 500 * 2 ** attempt) * (0.5 + Math.random())));
+      }
+    }
+  }
+
+  async attempt({ endpoint, model, content, maxOutputTokens = 4096, options = {}, provider, onDelta }) {
     let lastError;
     for (const relay of await this.relays()) {
       let candidates;
@@ -223,9 +255,18 @@ export class Consumer {
           && (!provider || descriptor.provider_signer === provider.toLowerCase()));
       }
       catch (error) { lastError = error; continue; }
-      // What the chain proves ranks first: recent fraud, verified probe failures, reputation; then price.
+      // What the chain proves ranks first (recent fraud, a suspected downgrade, verified probe failures); then
+      // Providers this Consumer is not already keeping busy; then reputation in coarse bands, shuffled within a
+      // band so that many Consumers spread over the network instead of all queueing at its single best Provider.
       const records = await this.reputations().catch(() => ({}));
-      candidates.sort((a, b) => compareRank(rankKey(a, records[a.provider_signer]), rankKey(b, records[b.provider_signer])));
+      const key = (descriptor) => {
+        const [fraud, downgrade, failure, volume] = rankKey(descriptor, records[descriptor.provider_signer]);
+        const busy = (this.inFlight.get(descriptor.provider_signer) || 0) >= Math.max(1, Number(descriptor.capacity) || 1) ? 1 : 0;
+        return [fraud, downgrade, busy, failure, -Math.floor(Math.log10(1 - volume)), Math.random()];
+      };
+      const keyed = candidates.map((descriptor) => [key(descriptor), descriptor]);
+      keyed.sort((a, b) => compareRank(a[0], b[0]));
+      candidates = keyed.map(([, descriptor]) => descriptor);
       for (const descriptor of candidates) {
         const prepared = prepareRequest({
           descriptor, deployment: this.deployment, keyPrivate: this.keyPrivate, relaySigner: relay.signer,
@@ -234,6 +275,8 @@ export class Consumer {
         let reply;
         const streamed = [];
         let final = null;
+        const signer = descriptor.provider_signer;
+        this.inFlight.set(signer, (this.inFlight.get(signer) || 0) + 1);
         try {
           reply = onDelta
             ? await this.streamJson(`${relay.url}/v11/requests`, { body: prepared.payload, ca: this.network.tls_ca, pin: relay.pin, onLine: (line) => {
@@ -248,11 +291,15 @@ export class Consumer {
         } catch (error) {
           // The request may have reached the Relay: never replay it elsewhere.
           throw Object.assign(new Error(`request outcome unknown: ${error.message}`), { code: "outcome_unknown", requestId: prepared.authorization.request_id });
+        } finally {
+          this.inFlight.set(signer, (this.inFlight.get(signer) || 1) - 1);
         }
         if (reply.streamed) {
           if (!final) throw Object.assign(new Error("stream ended without a result"), { code: "outcome_unknown" });
-          if (final.type === "error") throw Object.assign(new Error(final.error), { status: final.status, dispatched: final.dispatched });
-          reply = { status: 200, body: final };
+          // An error before dispatch (a busy Provider, say) is safe to fail over, exactly as without streaming.
+          reply = final.type === "error"
+            ? { status: final.status || 502, body: { error: final.error, dispatched: final.dispatched } }
+            : { status: 200, body: final };
         }
         if (reply.status === 200) {
           const opened = openResponse(prepared, reply.body, this.deployment, this.now());
@@ -263,7 +310,8 @@ export class Consumer {
           if (this.journalDir) recordRequest(this.journalDir, prepared, opened, relay.url);
           return opened;
         }
-        lastError = Object.assign(new Error(reply.body?.error || `Relay returned ${reply.status}`), { status: reply.status });
+        lastError = Object.assign(new Error(reply.body?.error || `Relay returned ${reply.status}`),
+          { status: reply.status, dispatched: reply.body?.dispatched });
         if (reply.body?.dispatched !== false || ![502, 503].includes(reply.status)) throw lastError;
       }
     }