@diffcp/core 0.1.0-canary.2

package/README.md

@@ -0,0 +1,47 @@
+# ☄️ Diffcp Core
+
+Diffcp (Differential Context Protocol) is the new standard to stream AI Agent state to the user interface. Is a
+lightweight alternative to any bespoke AI protocol currently in the industry. Purpose built to be versatile
+unopinionated and highly efficient (95%+ compression).
+
+- [Documentation](https://github.com/axelered/diffcp)
+- [GitHub](https://github.com/axelered/diffcp)
+- [Issues](https://github.com/axelered/diffcp/issues)
+
+## Install
+
+```shell
+npm i @diffcp/core
+```
+
+## Use
+
+Convert your existing APIs to a streaming endpoint with a **one line** and a simple function which
+**yield updated state objects**
+
+```ts
+export interface MessageType {
+	text: string
+}
+```
+
+```ts
+export async function* streamMessage(): AsyncIterable<MessageType> {
+	yield { text: 'This' }
+	yield { text: 'This is' }
+	// ...
+	yield { text: 'This is a stream message completed' }
+}
+
+export function GET() {
+	return new ObjectStreamResponse(streamMessage())
+}
+```
+
+On the client just **consume an updating state stream**
+
+```ts
+for await (const data of fetchObjectStream<MessageType>('/api')) {
+	// Consume data
+}
+```

package/dist/index.cjs

@@ -0,0 +1,411 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/index.ts
+var index_exports = {};
+__export(index_exports, {
+  NdJSONStreamResponse: () => NdJSONStreamResponse,
+  ObjectStreamResponse: () => ObjectStreamResponse,
+  PlainJsonError: () => PlainJsonError,
+  StreamEvent: () => StreamEvent,
+  StreamReinit: () => StreamReinit,
+  deflate: () => deflate,
+  diffApply: () => diffApply,
+  diffCreate: () => diffCreate,
+  fetchNdJSON: () => fetchNdJSON,
+  fetchObjectStream: () => fetchObjectStream,
+  inflate: () => inflate
+});
+module.exports = __toCommonJS(index_exports);
+
+// src/diff.ts
+function diffEncodePath(path) {
+  if (path.length === 0) return "";
+  return "/" + path.map((x) => (x + "").replace(/~/g, "~0").replace(/\//g, "~1")).join("/");
+}
+function diffDecodePath(path) {
+  if (path == "") {
+    return [];
+  } else if (!path.startsWith("/")) {
+    throw Error(`Invalid path string "${path}"`);
+  }
+  return path.substring(1).split("/").map((x) => x.replace(/~1/g, "/").replace(/~0/g, "~"));
+}
+function jsonTypeof(value) {
+  const t = typeof value;
+  if (t === "bigint") {
+    return "number";
+  } else if (t === "function" || t === "symbol") {
+    throw Error(`Unsupported value type "${t}"`);
+  } else if (t === "object" && Array.isArray(value)) {
+    return "array";
+  } else {
+    return t;
+  }
+}
+function diffCreateOp(diff, currentPath, from, to) {
+  if (from === to) {
+    return;
+  }
+  const tFrom = jsonTypeof(from);
+  const tTo = jsonTypeof(to);
+  if (tFrom !== tTo) {
+    diff.push(["s", diffEncodePath(currentPath), to]);
+  } else if (tFrom === "object") {
+    const fromKeys = new Set(Object.keys(from));
+    const toKeys = new Set(Object.keys(to));
+    const keys = /* @__PURE__ */ new Set([...fromKeys, ...toKeys]);
+    for (const key of keys) {
+      if (!toKeys.has(key)) {
+        diff.push(["d", diffEncodePath([...currentPath, key])]);
+      } else if (!fromKeys.has(key)) {
+        diff.push(["s", diffEncodePath([...currentPath, key]), to[key]]);
+      } else {
+        diffCreateOp(diff, [...currentPath, key], from[key], to[key]);
+      }
+    }
+  } else if (tFrom === "array") {
+    const lengthMax = Math.max(from.length, to.length);
+    const lengthMin = Math.min(from.length, to.length);
+    for (let i = 0; i < lengthMax; i++) {
+      if (i < lengthMin) {
+        diffCreateOp(diff, [...currentPath, i], from[i], to[i]);
+      } else if (from.length > to.length) {
+        diff.push(["d", diffEncodePath([...currentPath, i])]);
+      } else {
+        diff.push(["a", diffEncodePath([...currentPath, "-"]), to[i]]);
+      }
+    }
+  } else if (tFrom === "string") {
+    if (to.startsWith(from)) {
+      diff.push(["a", diffEncodePath([...currentPath, "-"]), to.substring(from.length)]);
+    } else {
+      diff.push(["s", diffEncodePath(currentPath), to]);
+    }
+  } else {
+    diff.push(["s", diffEncodePath(currentPath), to]);
+  }
+}
+function diffCreate(from, to) {
+  const diff = [];
+  diffCreateOp(diff, [], from, to);
+  return diff;
+}
+function diffApplyOp(currentPath, restPath, value, op) {
+  if (restPath.length === 0) {
+    if (op[0] !== "s") {
+      throw Error(`Cannot perform operation ${op[0]} at "${op[1]}"`);
+    } else {
+      return op[2];
+    }
+  }
+  const key = restPath[0];
+  if (key === "-" && (op[0] !== "a" || restPath.length > 1)) {
+    throw Error(`Cannot perform operation ${op[0]} for "-" indexes at "${op[1]}"`);
+  }
+  const nextCurrentPath = [...currentPath, key];
+  const nextRestPath = restPath.slice(1);
+  const isDelete = nextRestPath.length === 0 && op[0] === "d";
+  if (typeof value === "object" && Array.isArray(value)) {
+    if (key === "-") {
+      return [...value, op[2]];
+    } else {
+      const ix = Number.parseInt(key, 10);
+      if (Number.isNaN(ix) || ix < 0 || ix >= value.length) {
+        throw Error(`Invalid access index ${ix} for operation ${op[0]} at "${op[1]}"`);
+      }
+      if (isDelete) {
+        return value.filter((_, jx) => ix !== jx);
+      } else {
+        return value.map(
+          (v, jx) => ix !== jx ? v : diffApplyOp(nextCurrentPath, nextRestPath, v, op)
+        );
+      }
+    }
+  } else if (typeof value === "object") {
+    if (isDelete) {
+      return { ...value, [key]: void 0 };
+    } else {
+      return {
+        ...value,
+        [key]: diffApplyOp(nextCurrentPath, nextRestPath, value[key], op)
+      };
+    }
+  } else if (typeof value === "string") {
+    if (key === "-") {
+      return value + op[2];
+    }
+  }
+}
+function diffApply(value, diff) {
+  if (value === void 0 && diff.length === 0) {
+    throw Error(`Cannot apply empty diff without an object`);
+  }
+  for (const op of diff) {
+    const path = diffDecodePath(op[1]);
+    value = diffApplyOp([], path, value, op);
+  }
+  return value;
+}
+
+// src/ndjson.ts
+var NdJSONStreamResponse = class extends Response {
+  constructor(messages, init) {
+    const encoder = new TextEncoder();
+    let heartbeat = null;
+    const stream = new ReadableStream({
+      async start(controller) {
+        heartbeat = setInterval(() => {
+          controller.enqueue("\n");
+        }, init?.heartbeatMs ?? 15e3);
+        for await (const msg of messages) {
+          controller.enqueue(encoder.encode(JSON.stringify(msg) + "\n"));
+        }
+        clearInterval(heartbeat);
+        controller.close();
+      },
+      cancel() {
+        clearInterval(heartbeat);
+      }
+    });
+    super(stream, {
+      ...init,
+      headers: {
+        "Content-Type": "application/x-ndjson; charset=utf-8",
+        "Transfer-Encoding": "chunked",
+        "Cache-Control": "no-cache, no-transform",
+        Connection: "keep-alive",
+        "X-Accel-Buffering": "no",
+        // Disable nginx buffering
+        ...init?.headers
+      }
+    });
+  }
+};
+function parseContentType(value) {
+  const [typePart, ...paramParts] = value.split(";");
+  const type = typePart.trim().toLowerCase();
+  const parameters = {};
+  for (const part of paramParts) {
+    const [key, ...rest] = part.split("=");
+    if (!key || rest.length === 0) continue;
+    let val = rest.join("=").trim();
+    if (val.startsWith('"') && val.endsWith('"')) val = val.slice(1, -1);
+    parameters[key.trim().toLowerCase()] = val;
+  }
+  return [type, parameters["charset"] || "utf8"];
+}
+var PlainJsonError = class extends Error {
+  constructor(data) {
+    super("fetch failed: received plain application/json response");
+    this.data = data;
+  }
+};
+async function* fetchNdJSON(input, init) {
+  const res = await fetch(input, init);
+  if (!res.ok) {
+    throw new Error(`fetch failed: ${res.status} ${res.statusText}`);
+  } else if (!res.body) {
+    throw new Error(`fetch failed: empty request body`);
+  }
+  const ctHeader = res.headers.get("content-type");
+  const [type, encoding] = parseContentType(ctHeader ?? "");
+  if (type === "application/x-ndjson") {
+    const reader = res.body.getReader();
+    const decoder = new TextDecoder(encoding);
+    let buffer = "";
+    while (true) {
+      const { value, done } = await reader.read();
+      if (done) break;
+      buffer += decoder.decode(value, { stream: true });
+      const lines = buffer.split("\n");
+      buffer = lines.pop() ?? "";
+      for (const line of lines) {
+        const trimmed = line.trim();
+        if (!trimmed) continue;
+        const v = JSON.parse(trimmed);
+        init?.onLine?.(v);
+        yield v;
+      }
+    }
+    const tail = buffer.trim();
+    if (tail) {
+      const v = JSON.parse(tail);
+      init?.onLine?.(v);
+      yield v;
+    }
+  } else if (type === "application/json") {
+    throw new PlainJsonError(await res.json());
+  } else {
+    throw new Error(`fetch failed: invalid response content type ${ctHeader}`);
+  }
+}
+
+// src/zip.ts
+async function* deflate(messages) {
+  const objType = { init: 1, delta: 2, done: 3, event: 9 };
+  const patchType = { a: 1, d: 2, s: 3 };
+  const deltaDict = {};
+  let deltaOffset = 0;
+  for await (const msg of messages) {
+    if (msg.t === "delta") {
+      const delta = [];
+      for (const [typ, path, ...rest] of msg.d) {
+        delta.push([
+          patchType[typ],
+          // Previous Occurrence Compression
+          path in deltaDict ? deltaOffset - deltaDict[path] : path,
+          ...rest
+        ]);
+        deltaDict[path] = deltaOffset;
+        deltaOffset += 1;
+      }
+      yield [objType.delta, delta];
+    } else {
+      yield msg.d ? [objType[msg.t], msg.d] : [objType[msg.t]];
+    }
+  }
+}
+async function* inflate(compressed) {
+  const objType = { 1: "init", 2: "delta", 3: "done", 9: "event" };
+  const patchType = { 1: "a", 2: "d", 3: "s" };
+  const deltaDict = {};
+  const deltaDictInv = {};
+  let deltaOffset = 0;
+  for await (const rec of compressed) {
+    if (!Array.isArray(rec)) {
+      yield rec;
+      continue;
+    }
+    const [typ, data] = rec;
+    if (objType[typ] === "delta") {
+      const delta = [];
+      for (const [typ2, path, ...rest] of data) {
+        let decPath = path;
+        if (typeof path === "number") {
+          const oldOffset = deltaOffset - path;
+          decPath = deltaDict[oldOffset];
+          delete deltaDict[oldOffset];
+        }
+        if (decPath in deltaDictInv) delete deltaDict[deltaDictInv[decPath]];
+        deltaDictInv[decPath] = deltaOffset;
+        deltaDict[deltaOffset] = decPath;
+        deltaOffset += 1;
+        delta.push([patchType[typ2], decPath, ...rest]);
+      }
+      yield { t: "delta", d: delta };
+    } else {
+      yield { t: objType[typ], d: data };
+    }
+  }
+}
+
+// src/stream.ts
+var StreamEvent = class {
+  constructor(data) {
+    this.data = data;
+  }
+};
+var StreamReinit = class {
+  constructor(data) {
+    this.data = data;
+  }
+};
+var ObjectStreamResponse = class extends NdJSONStreamResponse {
+  constructor(stream, init) {
+    async function* wrapped() {
+      let last = void 0;
+      for await (const chunk of stream) {
+        if (chunk instanceof StreamEvent) {
+          yield { t: "event", d: chunk.data };
+          continue;
+        }
+        if (chunk instanceof StreamReinit) last = void 0;
+        const data = chunk instanceof StreamReinit ? chunk.data : chunk;
+        if (last === void 0) {
+          yield { t: "init", d: data };
+        } else {
+          yield { t: "delta", d: diffCreate(last, data) };
+        }
+        last = JSON.parse(JSON.stringify(data));
+      }
+      yield init?.sendDataOnDone ? { t: "done", d: last } : { t: "done" };
+    }
+    const compress = init?.compressed !== false;
+    const res = compress ? deflate(wrapped()) : wrapped();
+    super(res, {
+      ...init,
+      headers: {
+        ...init?.headers,
+        "X-Diff-Version": compress ? "1; compressed" : "1"
+      }
+    });
+  }
+};
+async function* fetchObjectStream(input, init) {
+  const { onEvent, ...ndInit } = init ?? {};
+  let done = false;
+  let dataValue = void 0;
+  try {
+    const res = fetchNdJSON(input, ndInit);
+    for await (const data of inflate(res)) {
+      init?.onFrame?.(data);
+      if (data.t === "init") {
+        dataValue = data.d;
+        init?.onData?.(dataValue);
+        yield dataValue;
+      } else if (data.t === "delta") {
+        dataValue = diffApply(dataValue, data.d);
+        init?.onData?.(dataValue);
+        yield dataValue;
+      } else if (data.t === "event") {
+        onEvent?.(data.d);
+      } else if (data.t === "done") {
+        if (data.d !== void 0) dataValue = data.d;
+        done = true;
+      }
+    }
+  } catch (error) {
+    if (error instanceof PlainJsonError && init?.fallbackPlainJson !== false) {
+      return error.data;
+    } else {
+      throw error;
+    }
+  }
+  if (!dataValue) {
+    throw Error(`Object stream ended without any data`);
+  } else if (!done) {
+    throw Error(`Object stream ended without done signal`);
+  }
+  return dataValue;
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  NdJSONStreamResponse,
+  ObjectStreamResponse,
+  PlainJsonError,
+  StreamEvent,
+  StreamReinit,
+  deflate,
+  diffApply,
+  diffCreate,
+  fetchNdJSON,
+  fetchObjectStream,
+  inflate
+});

package/dist/index.d.cts

@@ -0,0 +1,81 @@
+type ObjectPatchAdd = ['a', string, any];
+type ObjectPatchSet = ['s', string, any];
+type ObjectPatchRemove = ['d', string];
+type ObjectPatchOp = ObjectPatchAdd | ObjectPatchSet | ObjectPatchRemove;
+type ObjectPatchDiff = ObjectPatchOp[];
+/**
+ * Creates a set of diff operations to transition from Object to Object
+ */
+declare function diffCreate<T extends object>(from: T | undefined, to: T): ObjectPatchDiff;
+/**
+ * Apply a complete set of diff operations to an Object
+ */
+declare function diffApply<T extends object>(value: T | undefined, diff: ObjectPatchDiff): T;
+
+interface NDJSONStreamResponseInit extends ResponseInit {
+    heartbeatMs?: number;
+}
+/**
+ * New line delimited JSON response stream
+ */
+declare class NdJSONStreamResponse<M> extends Response {
+    constructor(messages: AsyncIterable<M>, init?: NDJSONStreamResponseInit);
+}
+declare class PlainJsonError extends Error {
+    readonly data: any;
+    constructor(data: any);
+}
+interface NdJSONFetchRequestInit<T> extends RequestInit {
+    onLine?: (value: T) => void;
+}
+/**
+ * New line delimited JSON fetch implementation
+ */
+declare function fetchNdJSON<T>(input: string | URL | Request, init?: NdJSONFetchRequestInit<T>): AsyncIterable<T>;
+
+type CompressedObjectPatchDiff = [1 | 3, number | string, any] | [2, number | string];
+type CompressedObjectStream<T extends object, E> = [1, T] | [2, CompressedObjectPatchDiff] | [3, T] | [9, E];
+declare function deflate<T extends object, E>(messages: AsyncIterable<ObjectStream<T, E>>): AsyncIterable<CompressedObjectStream<T, E>>;
+declare function inflate<T extends object, E>(compressed: AsyncIterable<CompressedObjectStream<T, E>>): AsyncIterable<ObjectStream<T, E>>;
+
+type ObjectStream<T extends object, E> = {
+    t: 'init';
+    d: T;
+} | {
+    t: 'delta';
+    d: ObjectPatchDiff;
+} | {
+    t: 'done';
+    d?: T;
+} | {
+    t: 'event';
+    d: E;
+};
+declare class StreamEvent<E> {
+    readonly data: E;
+    constructor(data: E);
+}
+declare class StreamReinit<T extends object> {
+    readonly data: T;
+    constructor(data: T);
+}
+type ObjectStreamIterable<T extends object, E> = AsyncIterable<T | StreamReinit<T> | StreamEvent<E>>;
+interface ObjectStreamResponseInit extends NDJSONStreamResponseInit {
+    sendDataOnDone?: boolean;
+    compressed?: boolean;
+}
+/**
+ * Differential Context Protocol Stream
+ */
+declare class ObjectStreamResponse<T extends object, E = any> extends NdJSONStreamResponse<ObjectStream<T, E> | CompressedObjectStream<T, E>> {
+    constructor(stream: ObjectStreamIterable<T, E>, init?: ObjectStreamResponseInit);
+}
+interface ObjectStreamRequestInit<T extends object, E> extends NdJSONFetchRequestInit<ObjectStream<T, E> | CompressedObjectStream<T, E>> {
+    onData?: (data: T) => void;
+    onEvent?: (event: E) => void;
+    onFrame?: (frame: ObjectStream<T, E>) => void;
+    fallbackPlainJson?: boolean;
+}
+declare function fetchObjectStream<T extends object, E = any>(input: string | URL | Request, init?: ObjectStreamRequestInit<T, E>): AsyncIterable<T>;
+
+export { type CompressedObjectPatchDiff, type CompressedObjectStream, type NDJSONStreamResponseInit, type NdJSONFetchRequestInit, NdJSONStreamResponse, type ObjectPatchAdd, type ObjectPatchDiff, type ObjectPatchOp, type ObjectPatchRemove, type ObjectPatchSet, type ObjectStream, type ObjectStreamIterable, type ObjectStreamRequestInit, ObjectStreamResponse, type ObjectStreamResponseInit, PlainJsonError, StreamEvent, StreamReinit, deflate, diffApply, diffCreate, fetchNdJSON, fetchObjectStream, inflate };

package/dist/index.d.ts

@@ -0,0 +1,81 @@
+type ObjectPatchAdd = ['a', string, any];
+type ObjectPatchSet = ['s', string, any];
+type ObjectPatchRemove = ['d', string];
+type ObjectPatchOp = ObjectPatchAdd | ObjectPatchSet | ObjectPatchRemove;
+type ObjectPatchDiff = ObjectPatchOp[];
+/**
+ * Creates a set of diff operations to transition from Object to Object
+ */
+declare function diffCreate<T extends object>(from: T | undefined, to: T): ObjectPatchDiff;
+/**
+ * Apply a complete set of diff operations to an Object
+ */
+declare function diffApply<T extends object>(value: T | undefined, diff: ObjectPatchDiff): T;
+
+interface NDJSONStreamResponseInit extends ResponseInit {
+    heartbeatMs?: number;
+}
+/**
+ * New line delimited JSON response stream
+ */
+declare class NdJSONStreamResponse<M> extends Response {
+    constructor(messages: AsyncIterable<M>, init?: NDJSONStreamResponseInit);
+}
+declare class PlainJsonError extends Error {
+    readonly data: any;
+    constructor(data: any);
+}
+interface NdJSONFetchRequestInit<T> extends RequestInit {
+    onLine?: (value: T) => void;
+}
+/**
+ * New line delimited JSON fetch implementation
+ */
+declare function fetchNdJSON<T>(input: string | URL | Request, init?: NdJSONFetchRequestInit<T>): AsyncIterable<T>;
+
+type CompressedObjectPatchDiff = [1 | 3, number | string, any] | [2, number | string];
+type CompressedObjectStream<T extends object, E> = [1, T] | [2, CompressedObjectPatchDiff] | [3, T] | [9, E];
+declare function deflate<T extends object, E>(messages: AsyncIterable<ObjectStream<T, E>>): AsyncIterable<CompressedObjectStream<T, E>>;
+declare function inflate<T extends object, E>(compressed: AsyncIterable<CompressedObjectStream<T, E>>): AsyncIterable<ObjectStream<T, E>>;
+
+type ObjectStream<T extends object, E> = {
+    t: 'init';
+    d: T;
+} | {
+    t: 'delta';
+    d: ObjectPatchDiff;
+} | {
+    t: 'done';
+    d?: T;
+} | {
+    t: 'event';
+    d: E;
+};
+declare class StreamEvent<E> {
+    readonly data: E;
+    constructor(data: E);
+}
+declare class StreamReinit<T extends object> {
+    readonly data: T;
+    constructor(data: T);
+}
+type ObjectStreamIterable<T extends object, E> = AsyncIterable<T | StreamReinit<T> | StreamEvent<E>>;
+interface ObjectStreamResponseInit extends NDJSONStreamResponseInit {
+    sendDataOnDone?: boolean;
+    compressed?: boolean;
+}
+/**
+ * Differential Context Protocol Stream
+ */
+declare class ObjectStreamResponse<T extends object, E = any> extends NdJSONStreamResponse<ObjectStream<T, E> | CompressedObjectStream<T, E>> {
+    constructor(stream: ObjectStreamIterable<T, E>, init?: ObjectStreamResponseInit);
+}
+interface ObjectStreamRequestInit<T extends object, E> extends NdJSONFetchRequestInit<ObjectStream<T, E> | CompressedObjectStream<T, E>> {
+    onData?: (data: T) => void;
+    onEvent?: (event: E) => void;
+    onFrame?: (frame: ObjectStream<T, E>) => void;
+    fallbackPlainJson?: boolean;
+}
+declare function fetchObjectStream<T extends object, E = any>(input: string | URL | Request, init?: ObjectStreamRequestInit<T, E>): AsyncIterable<T>;
+
+export { type CompressedObjectPatchDiff, type CompressedObjectStream, type NDJSONStreamResponseInit, type NdJSONFetchRequestInit, NdJSONStreamResponse, type ObjectPatchAdd, type ObjectPatchDiff, type ObjectPatchOp, type ObjectPatchRemove, type ObjectPatchSet, type ObjectStream, type ObjectStreamIterable, type ObjectStreamRequestInit, ObjectStreamResponse, type ObjectStreamResponseInit, PlainJsonError, StreamEvent, StreamReinit, deflate, diffApply, diffCreate, fetchNdJSON, fetchObjectStream, inflate };

package/dist/index.js

@@ -0,0 +1,374 @@
+// src/diff.ts
+function diffEncodePath(path) {
+  if (path.length === 0) return "";
+  return "/" + path.map((x) => (x + "").replace(/~/g, "~0").replace(/\//g, "~1")).join("/");
+}
+function diffDecodePath(path) {
+  if (path == "") {
+    return [];
+  } else if (!path.startsWith("/")) {
+    throw Error(`Invalid path string "${path}"`);
+  }
+  return path.substring(1).split("/").map((x) => x.replace(/~1/g, "/").replace(/~0/g, "~"));
+}
+function jsonTypeof(value) {
+  const t = typeof value;
+  if (t === "bigint") {
+    return "number";
+  } else if (t === "function" || t === "symbol") {
+    throw Error(`Unsupported value type "${t}"`);
+  } else if (t === "object" && Array.isArray(value)) {
+    return "array";
+  } else {
+    return t;
+  }
+}
+function diffCreateOp(diff, currentPath, from, to) {
+  if (from === to) {
+    return;
+  }
+  const tFrom = jsonTypeof(from);
+  const tTo = jsonTypeof(to);
+  if (tFrom !== tTo) {
+    diff.push(["s", diffEncodePath(currentPath), to]);
+  } else if (tFrom === "object") {
+    const fromKeys = new Set(Object.keys(from));
+    const toKeys = new Set(Object.keys(to));
+    const keys = /* @__PURE__ */ new Set([...fromKeys, ...toKeys]);
+    for (const key of keys) {
+      if (!toKeys.has(key)) {
+        diff.push(["d", diffEncodePath([...currentPath, key])]);
+      } else if (!fromKeys.has(key)) {
+        diff.push(["s", diffEncodePath([...currentPath, key]), to[key]]);
+      } else {
+        diffCreateOp(diff, [...currentPath, key], from[key], to[key]);
+      }
+    }
+  } else if (tFrom === "array") {
+    const lengthMax = Math.max(from.length, to.length);
+    const lengthMin = Math.min(from.length, to.length);
+    for (let i = 0; i < lengthMax; i++) {
+      if (i < lengthMin) {
+        diffCreateOp(diff, [...currentPath, i], from[i], to[i]);
+      } else if (from.length > to.length) {
+        diff.push(["d", diffEncodePath([...currentPath, i])]);
+      } else {
+        diff.push(["a", diffEncodePath([...currentPath, "-"]), to[i]]);
+      }
+    }
+  } else if (tFrom === "string") {
+    if (to.startsWith(from)) {
+      diff.push(["a", diffEncodePath([...currentPath, "-"]), to.substring(from.length)]);
+    } else {
+      diff.push(["s", diffEncodePath(currentPath), to]);
+    }
+  } else {
+    diff.push(["s", diffEncodePath(currentPath), to]);
+  }
+}
+function diffCreate(from, to) {
+  const diff = [];
+  diffCreateOp(diff, [], from, to);
+  return diff;
+}
+function diffApplyOp(currentPath, restPath, value, op) {
+  if (restPath.length === 0) {
+    if (op[0] !== "s") {
+      throw Error(`Cannot perform operation ${op[0]} at "${op[1]}"`);
+    } else {
+      return op[2];
+    }
+  }
+  const key = restPath[0];
+  if (key === "-" && (op[0] !== "a" || restPath.length > 1)) {
+    throw Error(`Cannot perform operation ${op[0]} for "-" indexes at "${op[1]}"`);
+  }
+  const nextCurrentPath = [...currentPath, key];
+  const nextRestPath = restPath.slice(1);
+  const isDelete = nextRestPath.length === 0 && op[0] === "d";
+  if (typeof value === "object" && Array.isArray(value)) {
+    if (key === "-") {
+      return [...value, op[2]];
+    } else {
+      const ix = Number.parseInt(key, 10);
+      if (Number.isNaN(ix) || ix < 0 || ix >= value.length) {
+        throw Error(`Invalid access index ${ix} for operation ${op[0]} at "${op[1]}"`);
+      }
+      if (isDelete) {
+        return value.filter((_, jx) => ix !== jx);
+      } else {
+        return value.map(
+          (v, jx) => ix !== jx ? v : diffApplyOp(nextCurrentPath, nextRestPath, v, op)
+        );
+      }
+    }
+  } else if (typeof value === "object") {
+    if (isDelete) {
+      return { ...value, [key]: void 0 };
+    } else {
+      return {
+        ...value,
+        [key]: diffApplyOp(nextCurrentPath, nextRestPath, value[key], op)
+      };
+    }
+  } else if (typeof value === "string") {
+    if (key === "-") {
+      return value + op[2];
+    }
+  }
+}
+function diffApply(value, diff) {
+  if (value === void 0 && diff.length === 0) {
+    throw Error(`Cannot apply empty diff without an object`);
+  }
+  for (const op of diff) {
+    const path = diffDecodePath(op[1]);
+    value = diffApplyOp([], path, value, op);
+  }
+  return value;
+}
+
+// src/ndjson.ts
+var NdJSONStreamResponse = class extends Response {
+  constructor(messages, init) {
+    const encoder = new TextEncoder();
+    let heartbeat = null;
+    const stream = new ReadableStream({
+      async start(controller) {
+        heartbeat = setInterval(() => {
+          controller.enqueue("\n");
+        }, init?.heartbeatMs ?? 15e3);
+        for await (const msg of messages) {
+          controller.enqueue(encoder.encode(JSON.stringify(msg) + "\n"));
+        }
+        clearInterval(heartbeat);
+        controller.close();
+      },
+      cancel() {
+        clearInterval(heartbeat);
+      }
+    });
+    super(stream, {
+      ...init,
+      headers: {
+        "Content-Type": "application/x-ndjson; charset=utf-8",
+        "Transfer-Encoding": "chunked",
+        "Cache-Control": "no-cache, no-transform",
+        Connection: "keep-alive",
+        "X-Accel-Buffering": "no",
+        // Disable nginx buffering
+        ...init?.headers
+      }
+    });
+  }
+};
+function parseContentType(value) {
+  const [typePart, ...paramParts] = value.split(";");
+  const type = typePart.trim().toLowerCase();
+  const parameters = {};
+  for (const part of paramParts) {
+    const [key, ...rest] = part.split("=");
+    if (!key || rest.length === 0) continue;
+    let val = rest.join("=").trim();
+    if (val.startsWith('"') && val.endsWith('"')) val = val.slice(1, -1);
+    parameters[key.trim().toLowerCase()] = val;
+  }
+  return [type, parameters["charset"] || "utf8"];
+}
+var PlainJsonError = class extends Error {
+  constructor(data) {
+    super("fetch failed: received plain application/json response");
+    this.data = data;
+  }
+};
+async function* fetchNdJSON(input, init) {
+  const res = await fetch(input, init);
+  if (!res.ok) {
+    throw new Error(`fetch failed: ${res.status} ${res.statusText}`);
+  } else if (!res.body) {
+    throw new Error(`fetch failed: empty request body`);
+  }
+  const ctHeader = res.headers.get("content-type");
+  const [type, encoding] = parseContentType(ctHeader ?? "");
+  if (type === "application/x-ndjson") {
+    const reader = res.body.getReader();
+    const decoder = new TextDecoder(encoding);
+    let buffer = "";
+    while (true) {
+      const { value, done } = await reader.read();
+      if (done) break;
+      buffer += decoder.decode(value, { stream: true });
+      const lines = buffer.split("\n");
+      buffer = lines.pop() ?? "";
+      for (const line of lines) {
+        const trimmed = line.trim();
+        if (!trimmed) continue;
+        const v = JSON.parse(trimmed);
+        init?.onLine?.(v);
+        yield v;
+      }
+    }
+    const tail = buffer.trim();
+    if (tail) {
+      const v = JSON.parse(tail);
+      init?.onLine?.(v);
+      yield v;
+    }
+  } else if (type === "application/json") {
+    throw new PlainJsonError(await res.json());
+  } else {
+    throw new Error(`fetch failed: invalid response content type ${ctHeader}`);
+  }
+}
+
+// src/zip.ts
+async function* deflate(messages) {
+  const objType = { init: 1, delta: 2, done: 3, event: 9 };
+  const patchType = { a: 1, d: 2, s: 3 };
+  const deltaDict = {};
+  let deltaOffset = 0;
+  for await (const msg of messages) {
+    if (msg.t === "delta") {
+      const delta = [];
+      for (const [typ, path, ...rest] of msg.d) {
+        delta.push([
+          patchType[typ],
+          // Previous Occurrence Compression
+          path in deltaDict ? deltaOffset - deltaDict[path] : path,
+          ...rest
+        ]);
+        deltaDict[path] = deltaOffset;
+        deltaOffset += 1;
+      }
+      yield [objType.delta, delta];
+    } else {
+      yield msg.d ? [objType[msg.t], msg.d] : [objType[msg.t]];
+    }
+  }
+}
+async function* inflate(compressed) {
+  const objType = { 1: "init", 2: "delta", 3: "done", 9: "event" };
+  const patchType = { 1: "a", 2: "d", 3: "s" };
+  const deltaDict = {};
+  const deltaDictInv = {};
+  let deltaOffset = 0;
+  for await (const rec of compressed) {
+    if (!Array.isArray(rec)) {
+      yield rec;
+      continue;
+    }
+    const [typ, data] = rec;
+    if (objType[typ] === "delta") {
+      const delta = [];
+      for (const [typ2, path, ...rest] of data) {
+        let decPath = path;
+        if (typeof path === "number") {
+          const oldOffset = deltaOffset - path;
+          decPath = deltaDict[oldOffset];
+          delete deltaDict[oldOffset];
+        }
+        if (decPath in deltaDictInv) delete deltaDict[deltaDictInv[decPath]];
+        deltaDictInv[decPath] = deltaOffset;
+        deltaDict[deltaOffset] = decPath;
+        deltaOffset += 1;
+        delta.push([patchType[typ2], decPath, ...rest]);
+      }
+      yield { t: "delta", d: delta };
+    } else {
+      yield { t: objType[typ], d: data };
+    }
+  }
+}
+
+// src/stream.ts
+var StreamEvent = class {
+  constructor(data) {
+    this.data = data;
+  }
+};
+var StreamReinit = class {
+  constructor(data) {
+    this.data = data;
+  }
+};
+var ObjectStreamResponse = class extends NdJSONStreamResponse {
+  constructor(stream, init) {
+    async function* wrapped() {
+      let last = void 0;
+      for await (const chunk of stream) {
+        if (chunk instanceof StreamEvent) {
+          yield { t: "event", d: chunk.data };
+          continue;
+        }
+        if (chunk instanceof StreamReinit) last = void 0;
+        const data = chunk instanceof StreamReinit ? chunk.data : chunk;
+        if (last === void 0) {
+          yield { t: "init", d: data };
+        } else {
+          yield { t: "delta", d: diffCreate(last, data) };
+        }
+        last = JSON.parse(JSON.stringify(data));
+      }
+      yield init?.sendDataOnDone ? { t: "done", d: last } : { t: "done" };
+    }
+    const compress = init?.compressed !== false;
+    const res = compress ? deflate(wrapped()) : wrapped();
+    super(res, {
+      ...init,
+      headers: {
+        ...init?.headers,
+        "X-Diff-Version": compress ? "1; compressed" : "1"
+      }
+    });
+  }
+};
+async function* fetchObjectStream(input, init) {
+  const { onEvent, ...ndInit } = init ?? {};
+  let done = false;
+  let dataValue = void 0;
+  try {
+    const res = fetchNdJSON(input, ndInit);
+    for await (const data of inflate(res)) {
+      init?.onFrame?.(data);
+      if (data.t === "init") {
+        dataValue = data.d;
+        init?.onData?.(dataValue);
+        yield dataValue;
+      } else if (data.t === "delta") {
+        dataValue = diffApply(dataValue, data.d);
+        init?.onData?.(dataValue);
+        yield dataValue;
+      } else if (data.t === "event") {
+        onEvent?.(data.d);
+      } else if (data.t === "done") {
+        if (data.d !== void 0) dataValue = data.d;
+        done = true;
+      }
+    }
+  } catch (error) {
+    if (error instanceof PlainJsonError && init?.fallbackPlainJson !== false) {
+      return error.data;
+    } else {
+      throw error;
+    }
+  }
+  if (!dataValue) {
+    throw Error(`Object stream ended without any data`);
+  } else if (!done) {
+    throw Error(`Object stream ended without done signal`);
+  }
+  return dataValue;
+}
+export {
+  NdJSONStreamResponse,
+  ObjectStreamResponse,
+  PlainJsonError,
+  StreamEvent,
+  StreamReinit,
+  deflate,
+  diffApply,
+  diffCreate,
+  fetchNdJSON,
+  fetchObjectStream,
+  inflate
+};

package/package.json

@@ -0,0 +1,56 @@
+{
+  "name": "@diffcp/core",
+  "version": "0.1.0-canary.2",
+  "description": "Open-source standard for connecting AI user interfaces to backend systems",
+  "author": {
+    "name": "Axelered AI",
+    "email": "dev@axelered.com",
+    "url": "https://axelered.com/"
+  },
+  "contributors": [
+    {
+      "name": "Borut Svara",
+      "email": "borut@axelered.com",
+      "url": "https://svara.io/"
+    }
+  ],
+  "keywords": [
+    "ai",
+    "protocol",
+    "axelered"
+  ],
+  "homepage": "https://github.com/axelered/diffcp",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/axelered/diffcp.git"
+  },
+  "bugs": {
+    "url": "https://github.com/axelered/diffcp/issues",
+    "email": "dev@axelered.com"
+  },
+  "license": "MIT",
+  "scripts": {
+    "build": "tsup",
+    "dev": "tsup --watch",
+    "test": "vitest run"
+  },
+  "type": "module",
+  "sideEffects": false,
+  "types": "dist/index.d.ts",
+  "main": "dist/index.cjs",
+  "module": "dist/index.js",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "require": "./dist/index.cjs",
+      "import": "./dist/index.js"
+    }
+  },
+  "files": [
+    "dist",
+    "README.md"
+  ],
+  "publishConfig": {
+    "access": "public"
+  }
+}