@bothat-io/molenkopf 0.1.2

package/packages/core/src/pipeline/json-string-spans.ts

@@ -0,0 +1,143 @@
+export type JsonStringSpan = { start: number; end: number; value: string; key?: string };
+export type JsonStringReplacement = { start: number; end: number; value: string };
+
+type Frame = { type: "object"; state: "key" | "colon" | "value" | "comma"; key?: string } | { type: "array"; state: "value" | "comma" };
+const MAX_SCAN_DEPTH = 1000;
+
+export function replaceJsonStrings(text: string, replacements: JsonStringReplacement[]): string {
+  if (!replacements.length) return text;
+  const ordered = [...replacements].sort((a, b) => a.start - b.start);
+  let out = "";
+  let offset = 0;
+  for (const item of ordered) {
+    out += text.slice(offset, item.start) + JSON.stringify(item.value);
+    offset = item.end;
+  }
+  return out + text.slice(offset);
+}
+
+export function scanJsonStringValues(text: string): JsonStringSpan[] | undefined {
+  if (exceedsScanDepth(text)) return [];
+  try {
+    JSON.parse(text);
+  } catch {
+    return undefined;
+  }
+  const spans: JsonStringSpan[] = [];
+  const stack: Frame[] = [];
+  let i = skipWs(text, 0);
+  consumeValue(undefined);
+  return spans;
+
+  function consumeValue(key: string | undefined) {
+    i = skipWs(text, i);
+    const char = text[i];
+    if (char === '"') {
+      const token = readString(text, i);
+      spans.push({ start: i, end: token.end, value: token.value, key });
+      i = token.end;
+      return;
+    }
+    if (char === "{") return consumeObject();
+    if (char === "[") return consumeArray();
+    while (i < text.length && !/[\s,\]}]/.test(text[i])) i++;
+  }
+
+  function consumeObject() {
+    stack.push({ type: "object", state: "key" });
+    i++;
+    while (i < text.length) {
+      i = skipWs(text, i);
+      const frame = stack[stack.length - 1];
+      if (!frame || frame.type !== "object") return;
+      if (text[i] === "}") {
+        stack.pop();
+        i++;
+        return;
+      }
+      if (frame.state === "comma") {
+        if (text[i] === ",") {
+          frame.state = "key";
+          i++;
+          continue;
+        }
+        return;
+      }
+      if (frame.state === "key") {
+        const token = readString(text, i);
+        frame.key = token.value;
+        frame.state = "colon";
+        i = token.end;
+        continue;
+      }
+      if (frame.state === "colon") {
+        if (text[i] !== ":") return;
+        frame.state = "value";
+        i++;
+        continue;
+      }
+      consumeValue(frame.key);
+      frame.state = "comma";
+    }
+  }
+
+  function consumeArray() {
+    stack.push({ type: "array", state: "value" });
+    i++;
+    while (i < text.length) {
+      i = skipWs(text, i);
+      const frame = stack[stack.length - 1];
+      if (!frame || frame.type !== "array") return;
+      if (text[i] === "]") {
+        stack.pop();
+        i++;
+        return;
+      }
+      if (frame.state === "comma") {
+        if (text[i] === ",") {
+          frame.state = "value";
+          i++;
+          continue;
+        }
+        return;
+      }
+      consumeValue(undefined);
+      frame.state = "comma";
+    }
+  }
+}
+
+function skipWs(text: string, index: number): number {
+  while (index < text.length && /\s/.test(text[index])) index++;
+  return index;
+}
+
+function readString(text: string, start: number): { value: string; end: number } {
+  let end = start + 1;
+  while (end < text.length) {
+    if (text[end] === '"' && !isEscaped(text, end)) break;
+    end++;
+  }
+  end++;
+  return { value: JSON.parse(text.slice(start, end)), end };
+}
+
+function isEscaped(text: string, quoteIndex: number): boolean {
+  let slashes = 0;
+  for (let i = quoteIndex - 1; i >= 0 && text[i] === "\\"; i--) slashes++;
+  return slashes % 2 === 1;
+}
+
+function exceedsScanDepth(text: string): boolean {
+  let depth = 0;
+  let inString = false;
+  for (let i = 0; i < text.length; i++) {
+    const char = text[i];
+    if (char === '"' && !isEscaped(text, i)) inString = !inString;
+    if (inString) continue;
+    if (char === "{" || char === "[") depth++;
+    else if (char === "}" || char === "]") depth--;
+    if (depth > MAX_SCAN_DEPTH) return true;
+  }
+  return false;
+}

package/packages/core/src/pipeline/openai-request-rewriter.ts

@@ -0,0 +1,66 @@
+import { compressContext } from "../compression/context-compressor.ts";
+import { redactSecrets } from "../security/secret-redactor.ts";
+import { RetrievalStore } from "../store/retrieval-store.ts";
+import { estimateTokens } from "../utils/tokens.ts";
+import { replaceJsonStrings, scanJsonStringValues, type JsonStringReplacement } from "./json-string-spans.ts";
+
+export type RewriteAudit = {
+  compressedItems: number;
+  estimatedOriginalTokens: number;
+  estimatedCompressedTokens: number;
+  estimatedSavedTokens: number;
+  redactedSecrets: number;
+  retrievalIds: string[];
+  compressorsUsed: string[];
+  warnings: string[];
+};
+
+export type CompressResult = { body: string; compressedItems: number; savedTokens: number; redactedSecrets: number; retrievalIds: string[]; compressorsUsed: string[] };
+
+export async function rewriteOpenAiJsonBody(body: string, store: RetrievalStore, requestId?: string, options: { compress?: boolean } = {}): Promise<{ body: string; audit: RewriteAudit }> {
+  const redacted = redactSecrets(body);
+  const compressed = await compressJsonBody(redacted.text, store, requestId, options.compress !== false);
+  const audit = emptyAudit(body, redacted.redactions.length + compressed.redactedSecrets);
+  audit.compressedItems = compressed.compressedItems;
+  audit.retrievalIds = compressed.retrievalIds;
+  audit.compressorsUsed = compressed.compressorsUsed;
+  audit.estimatedCompressedTokens = estimateTokens(compressed.body);
+  audit.estimatedSavedTokens = compressed.savedTokens;
+  return { body: compressed.body, audit };
+}
+
+// Compression only (no redaction), so the middleware pipeline can run redaction
+// and compression as separate, individually-toggleable steps.
+export async function compressJsonBody(text: string, store: RetrievalStore, requestId?: string, compress = true): Promise<CompressResult> {
+  const acc: CompressResult = { body: text, compressedItems: 0, savedTokens: 0, redactedSecrets: 0, retrievalIds: [], compressorsUsed: [] };
+  if (!compress) return acc;
+  const spans = scanJsonStringValues(text);
+  if (!spans) return acc;
+  const replacements: JsonStringReplacement[] = [];
+  for (const span of spans) {
+    if (span.value.length < 2000) continue;
+    const result = await compressContext(span.value, store, requestId);
+    acc.redactedSecrets += result.redactedSecrets;
+    if (!result.compressed) continue;
+    acc.compressedItems++;
+    acc.savedTokens += Math.max(0, estimateTokens(span.value) - estimateTokens(result.text));
+    if (result.retrievalId) acc.retrievalIds.push(result.retrievalId);
+    if (result.compressorName) acc.compressorsUsed.push(result.compressorName);
+    replacements.push({ start: span.start, end: span.end, value: result.text });
+  }
+  if (replacements.length) acc.body = replaceJsonStrings(text, replacements);
+  return acc;
+}
+
+function emptyAudit(body: string, redactedSecrets: number): RewriteAudit {
+  return {
+    compressedItems: 0,
+    estimatedOriginalTokens: estimateTokens(body),
+    estimatedCompressedTokens: estimateTokens(body),
+    estimatedSavedTokens: 0,
+    redactedSecrets,
+    retrievalIds: [],
+    compressorsUsed: [],
+    warnings: []
+  };
+}

package/packages/core/src/plugins/builtin-plugin-descriptors.ts

@@ -0,0 +1,10 @@
+import { descriptor as contextCompressorDescriptor } from "../../../plugins/context-compressor-plugin/descriptor.ts";
+import { descriptor as obsidianGraphDescriptor } from "../../../plugins/obsidian-graph-plugin/descriptor.ts";
+import type { PluginDescriptor } from "./plugin-descriptor.ts";
+
+export { contextCompressorDescriptor, obsidianGraphDescriptor };
+
+export const builtinPluginDescriptors: PluginDescriptor[] = [
+  contextCompressorDescriptor,
+  obsidianGraphDescriptor
+];

package/packages/core/src/plugins/builtin-plugin-modules.ts

@@ -0,0 +1,9 @@
+import { plugin as contextCompressorPlugin } from "../../../plugins/context-compressor-plugin/plugin.ts";
+import { plugin as obsidianGraphPlugin } from "../../../plugins/obsidian-graph-plugin/plugin.ts";
+import type { MolenkopfPluginModule } from "./plugin-api.ts";
+import { contextCompressorDescriptor, obsidianGraphDescriptor } from "./builtin-plugin-descriptors.ts";
+
+export const builtinPluginModules: Record<string, MolenkopfPluginModule> = {
+  [contextCompressorDescriptor.id]: contextCompressorPlugin,
+  [obsidianGraphDescriptor.id]: obsidianGraphPlugin
+};

package/packages/core/src/plugins/plugin-api.ts

@@ -0,0 +1,96 @@
+import type { AuditManifest } from "../manifest/audit-store.ts";
+import type { MemoryGraph } from "../memory/memory-graph.ts";
+
+export type PluginJson = Record<string, unknown>;
+
+export type PluginBlock = {
+  status: number;
+  error: string;
+};
+
+export type PluginUsage = {
+  requests: number;
+  inputTokens: number;
+  outputTokens: number;
+};
+
+export type PluginMetrics = {
+  redactedSecrets: number;
+  compressedItems: number;
+  savedTokens: number;
+  retrievalIds: string[];
+  compressorsUsed: string[];
+};
+
+export type PluginLifecycleContext = {
+  pluginId: string;
+  dataDir?: string;
+  now: () => Date;
+  note: (message: string) => void;
+  port?: number;
+  reason?: string;
+};
+
+export type PluginRequestContext = {
+  requestId: string;
+  method: string;
+  path: string;
+  consumerId: string;
+  providerId: string;
+  body: string;
+  usageOf: (consumerId: string) => PluginUsage;
+  note: (message: string) => void;
+};
+
+export type PluginRequestResult = Partial<PluginMetrics> & {
+  body?: string;
+  providerId?: string;
+  block?: PluginBlock;
+  notes?: string[];
+  audit?: PluginJson[];
+  events?: PluginJson[];
+};
+
+export type PluginAuditContext = {
+  requestId: string;
+  providerId: string;
+  statusCode: number;
+  manifest: PluginJson;
+  note: (message: string) => void;
+};
+
+export type PluginEventContext = {
+  event: string;
+  data: PluginJson;
+  emit: (event: string, data: PluginJson) => void;
+};
+
+export type PluginDataContext = {
+  canManage: boolean;
+  userId?: string;
+  teamIds: string[];
+  scope: string;
+  plugin: PluginJson;
+  scopes: string[];
+  manifests: AuditManifest[];
+  memoryGraph?: MemoryGraph;
+};
+
+export type PluginRuntimeContext = {
+  pluginId: string;
+  dataDir?: string;
+  storage?: unknown;
+  now: () => Date;
+};
+
+export type MolenkopfPluginModule = {
+  onBoot?: (ctx: PluginLifecycleContext, runtime: PluginRuntimeContext) => void | Promise<void>;
+  onStart?: (ctx: PluginLifecycleContext, runtime: PluginRuntimeContext) => void | Promise<void>;
+  onEnable?: (ctx: PluginLifecycleContext, runtime: PluginRuntimeContext) => void | Promise<void>;
+  onDisable?: (ctx: PluginLifecycleContext, runtime: PluginRuntimeContext) => void | Promise<void>;
+  onRequest?: (ctx: PluginRequestContext, runtime: PluginRuntimeContext) => PluginRequestResult | void | Promise<PluginRequestResult | void>;
+  onAudit?: (ctx: PluginAuditContext, runtime: PluginRuntimeContext) => void | Promise<void>;
+  onEvent?: (ctx: PluginEventContext, runtime: PluginRuntimeContext) => void | Promise<void>;
+  getData?: (ctx: PluginDataContext, runtime: PluginRuntimeContext) => PluginJson | Promise<PluginJson>;
+  onStop?: (ctx: PluginLifecycleContext, runtime: PluginRuntimeContext) => void | Promise<void>;
+};

package/packages/core/src/plugins/plugin-catalog.ts

@@ -0,0 +1,42 @@
+import { builtinPluginDescriptors, type PluginCategory, type PluginDataScope, type PluginRuntimeHook, type PluginTrafficAccess, type PluginType } from "./plugin-descriptor.ts";
+import type { PluginPermission } from "./plugin-sdk.ts";
+
+export type MolenkopfPlugin = {
+  id: string;
+  name: string;
+  type: PluginType;
+  category: PluginCategory;
+  description: string;
+  traffic: PluginTrafficAccess;
+  enabledByDefault: boolean;
+  canToggle: boolean;
+  permissions: PluginPermission[];
+  hooks: PluginRuntimeHook[];
+  modulePath?: string;
+  pagePath?: string;
+  dataPath?: string;
+  dataScopes?: PluginDataScope[];
+  pipelineIndex?: number;
+};
+
+export const pluginCatalog: MolenkopfPlugin[] = builtinPluginDescriptors.map((plugin) => ({
+  id: plugin.id,
+  name: plugin.name,
+  type: plugin.type,
+  category: plugin.category,
+  description: plugin.description,
+  traffic: { reads: [...plugin.traffic.reads], mutates: [...plugin.traffic.mutates] },
+  permissions: [...plugin.permissions],
+  hooks: [...plugin.hooks],
+  modulePath: plugin.modulePath,
+  enabledByDefault: plugin.toggle.defaultEnabled,
+  canToggle: plugin.toggle.canDisable,
+  pagePath: plugin.workspace?.pagePath,
+  dataPath: plugin.workspace?.dataPath,
+  dataScopes: plugin.workspace ? [...plugin.workspace.dataScopes] : undefined,
+  pipelineIndex: plugin.pipelineIndex
+}));
+
+export function findPlugin(id: string): MolenkopfPlugin | undefined {
+  return pluginCatalog.find((plugin) => plugin.id === id);
+}

package/packages/core/src/plugins/plugin-descriptor.ts

@@ -0,0 +1,51 @@
+import type { PluginPermission } from "./plugin-sdk.ts";
+import { builtinPluginDescriptors as descriptors } from "./builtin-plugin-descriptors.ts";
+import { staticPluginPipeline } from "./static-pipeline.ts";
+
+export type PluginCategory = "safety" | "compression" | "storage" | "events" | "routing" | "visualization";
+export type PluginDataScope = "metrics" | "audit-summary" | "requests" | "memory-graph";
+export type PluginType = "observer" | "classifier" | "redactor" | "transformer" | "retriever" | "router" | "auditor" | "stream-filter";
+export type PluginTrafficMutation = "none" | "mask" | "transform" | "augment-context" | "route" | "block" | "audit-log" | "event-filter";
+export type PluginRuntimeHook =
+  | "request:metadata"
+  | "request:body:rewrite"
+  | "audit:manifest"
+  | "events:lifecycle"
+  | "provider:route"
+  | "workspace:local-page";
+
+export type PluginTogglePolicy = {
+  defaultEnabled: boolean;
+  canDisable: true;
+};
+
+export type PluginWorkspace = {
+  pagePath: string;
+  dataPath: string;
+  dataScopes: PluginDataScope[];
+};
+
+export type PluginTrafficAccess = {
+  reads: ("metadata" | "redacted-body" | "body" | "audit" | "events")[];
+  mutates: PluginTrafficMutation[];
+};
+
+export type PluginDescriptor = {
+  id: string;
+  name: string;
+  type: PluginType;
+  category: PluginCategory;
+  description: string;
+  traffic: PluginTrafficAccess;
+  permissions: PluginPermission[];
+  hooks: PluginRuntimeHook[];
+  toggle: PluginTogglePolicy;
+  modulePath?: string;
+  workspace?: PluginWorkspace;
+  pipelineIndex?: number;
+};
+
+export const builtinPluginDescriptors: PluginDescriptor[] = descriptors.map((plugin) => {
+  const pipelineIndex = (staticPluginPipeline as readonly string[]).indexOf(plugin.id);
+  return pipelineIndex >= 0 ? { ...plugin, pipelineIndex } : plugin;
+});

package/packages/core/src/plugins/plugin-sdk.ts

@@ -0,0 +1,47 @@
+import type { MolenkopfPluginModule } from "./plugin-api.ts";
+
+export type PluginPermission =
+  | "metadata:read"
+  | "body:read"
+  | "body:write"
+  | "audit:read"
+  | "audit:write"
+  | "events:write"
+  | "provider:write";
+
+export type LocalPlugin = {
+  name: string;
+  permissions: PluginPermission[];
+  module: MolenkopfPluginModule;
+};
+
+export function createLocalPluginRegistry() {
+  const plugins = new Map<string, LocalPlugin>();
+  return {
+    register(plugin: LocalPlugin) {
+      if (!/^[a-z0-9-]+$/.test(plugin.name)) throw new Error("invalid plugin name");
+      if (plugin.permissions.length === 0) throw new Error("plugin permissions required");
+      if (!hasRuntimeHook(plugin.module)) throw new Error("plugin runtime hook required");
+      if (plugins.has(plugin.name)) throw new Error(`duplicate plugin: ${plugin.name}`);
+      plugins.set(plugin.name, deepFreeze({ ...plugin, permissions: [...plugin.permissions], module: { ...plugin.module } }));
+    },
+    registerRemote(_url: string): never {
+      throw new Error("remote plugins are disabled");
+    },
+    get(name: string) {
+      return plugins.get(name);
+    },
+    names() {
+      return [...plugins.keys()];
+    }
+  };
+}
+
+function deepFreeze<T extends object>(value: T): T {
+  for (const item of Object.values(value)) if (item && typeof item === "object") deepFreeze(item as object);
+  return Object.freeze(value);
+}
+
+function hasRuntimeHook(module: MolenkopfPluginModule): boolean {
+  return Boolean(module.onBoot || module.onStart || module.onEnable || module.onDisable || module.onRequest || module.onAudit || module.onEvent || module.getData || module.onStop);
+}

package/packages/core/src/plugins/static-pipeline.ts

@@ -0,0 +1,5 @@
+export const staticPluginPipeline = [
+  "context-compressor-plugin"
+] as const;
+
+export type StaticPluginName = typeof staticPluginPipeline[number];

package/packages/core/src/profiles/profile-router.ts

@@ -0,0 +1,45 @@
+export type UpstreamProfile = {
+  name: string;
+  target: string;
+  healthy?: boolean;
+  envKey?: string;
+  budgetTokens?: number;
+  usedTokens?: number;
+};
+
+export type RoutingConfig = {
+  mode: "fixed" | "manual" | "failover";
+  profile?: string;
+  profiles: UpstreamProfile[];
+};
+
+export function chooseProfile(config: RoutingConfig): UpstreamProfile {
+  if (config.mode === "failover") {
+    const profile = config.profiles.find((item) => item.healthy !== false && hasBudget(item));
+    if (profile) return profile;
+    throw new Error("no healthy profile with remaining budget");
+  }
+  const name = config.profile;
+  if (!name) throw new Error(`explicit profile required for ${config.mode} routing`);
+  const profile = config.profiles.find((item) => item.name === name);
+  if (!profile) throw new Error(`unknown profile: ${name}`);
+  if (!hasBudget(profile)) throw new Error(`budget exhausted: ${profile.name}`);
+  return profile;
+}
+
+export function readCredential(profile: Pick<UpstreamProfile, "envKey">, env: Record<string, string | undefined> = process.env): string | undefined {
+  return profile.envKey ? env[profile.envKey] : undefined;
+}
+
+export function healthSummary(profiles: UpstreamProfile[]) {
+  return profiles.map((item) => ({
+    name: item.name,
+    target: item.target,
+    healthy: item.healthy !== false,
+    budgetRemaining: item.budgetTokens === undefined ? undefined : Math.max(0, item.budgetTokens - (item.usedTokens ?? 0))
+  }));
+}
+
+function hasBudget(profile: UpstreamProfile): boolean {
+  return profile.budgetTokens === undefined || (profile.usedTokens ?? 0) < profile.budgetTokens;
+}