@pyxmate/memory 0.3.0-beta → 0.4.2

package/dist/{chunk-L62JXWDM.mjs → chunk-AXTWVLDO.mjs}

@@ -1,6 +1,6 @@
 import {
   MemoryClient
-} from "./chunk-FEOGTCSW.mjs";
+} from "./chunk-YL5QMN6G.mjs";
 
 // ../dashboard/src/aggregations/consolidation-analytics.ts
 function analyzeConsolidationLog(entries) {

package/dist/{chunk-FEOGTCSW.mjs → chunk-YL5QMN6G.mjs}

@@ -94,6 +94,13 @@ var MemoryClient = class {
     return this.fetchApi(`/api/memory/entries${qs ? `?${qs}` : ""}`);
   }
   // --- Additional endpoints ---
+  /**
+   * Ingest a file with optional two-phase enrichment.
+   *
+   * Without enrichment callbacks: standard single-phase ingest (backwards compatible).
+   * With enrichment callbacks: fetches extracted images, calls describeImage for each,
+   * optionally extracts entities, then submits enrichment data back to the server.
+   */
   async ingestFile(file, options) {
     const formData = new FormData();
     formData.append("file", file);
@@ -105,7 +112,64 @@ var MemoryClient = class {
       body: formData,
       headers: this._authHeaders
     });
-    return this.parseApiResponse(res);
+    const result = await this.parseApiResponse(res);
+    if (result.enrichment && options?.enrichment?.describeImage) {
+      const { fileId, token, expiresAt, images } = result.enrichment;
+      const descriptions = [];
+      const CONCURRENCY = 5;
+      for (let i = 0; i < images.length; i += CONCURRENCY) {
+        const batch = images.slice(i, i + CONCURRENCY);
+        const batchResults = await Promise.all(
+          batch.map(async (imageMeta) => {
+            const imageRes = await fetch(
+              `${this.baseUrl}/api/memory/files/${fileId}/images/${imageMeta.imageId}?token=${encodeURIComponent(token)}`,
+              { headers: this._authHeaders }
+            );
+            if (!imageRes.ok) {
+              throw new MemoryServerError(
+                `Failed to fetch image ${imageMeta.imageId}: ${imageRes.status}`,
+                imageRes.status
+              );
+            }
+            const imageBuffer = await imageRes.arrayBuffer();
+            const description = await options.enrichment.describeImage(imageBuffer, imageMeta);
+            return { imageId: imageMeta.imageId, description };
+          })
+        );
+        descriptions.push(...batchResults);
+      }
+      let entities;
+      let relationships;
+      if (options.enrichment.extractEntities && descriptions.length > 0) {
+        const extracted = await options.enrichment.extractEntities(
+          descriptions.map((d) => d.description)
+        );
+        entities = extracted.entities;
+        relationships = extracted.relationships;
+      }
+      const enrichTokenHeader = `${token}:${expiresAt}`;
+      const enrichRes = await fetch(`${this.baseUrl}/api/memory/files/${fileId}/enrich`, {
+        method: "POST",
+        headers: {
+          "Content-Type": "application/json",
+          "X-Enrichment-Token": enrichTokenHeader,
+          ...this._authHeaders
+        },
+        body: JSON.stringify({
+          imageDescriptions: descriptions,
+          entities,
+          relationships
+        })
+      });
+      if (!enrichRes.ok) {
+        const body = await enrichRes.json().catch(() => ({}));
+        throw new MemoryServerError(
+          body.error ?? `Enrichment failed: ${enrichRes.status}`,
+          enrichRes.status
+        );
+      }
+    }
+    return result;
   }
   /** @deprecated Use {@link list} instead. Kept for backwards compatibility. */
   async listEntries(params) {

package/dist/dashboard.mjs

@@ -11,8 +11,8 @@ import {
   toGraphologyFormat,
   transformGraphData,
   unreachableHealth
-} from "./chunk-L62JXWDM.mjs";
-import "./chunk-FEOGTCSW.mjs";
+} from "./chunk-AXTWVLDO.mjs";
+import "./chunk-YL5QMN6G.mjs";
 export {
   DashboardClient,
   Poller,

package/dist/index.d.ts

@@ -1,4 +1,4 @@
-import { StoreInput as StoreInput$1, MemoryEntry as MemoryEntry$1, MemorySearchParams as MemorySearchParams$1, MemorySearchResult as MemorySearchResult$1, MemoryType as MemoryType$1, MemoryStats as MemoryStats$1, GraphNode as GraphNode$1, GraphTraversalResult as GraphTraversalResult$1 } from '@pyx-memory/shared';
+import { StoreInput as StoreInput$1, MemoryEntry as MemoryEntry$1, MemorySearchParams as MemorySearchParams$1, MemorySearchResult as MemorySearchResult$1, MemoryType as MemoryType$1, MemoryStats as MemoryStats$1, ExtractedImageMeta, IngestEntity as IngestEntity$1, IngestRelationship as IngestRelationship$1, FileIngestResult, GraphNode as GraphNode$1, GraphTraversalResult as GraphTraversalResult$1 } from '@pyx-memory/shared';
 
 /** Parameters for paginated entry listing. */
 interface MemoryListParams {
@@ -65,14 +65,16 @@ interface ExtendedMemoryInterface extends MemoryInterface {
     deleteBySource(source: string): Promise<number>;
 }
 
-interface IngestionResult {
-    filename: string;
-    fileType: string;
-    chunks: number;
-    entryIds: string[];
-    totalCharacters: number;
+/** Callbacks for two-phase file enrichment. */
+interface EnrichmentCallbacks {
+    /** Describe an image using LLM vision. Receives the raw image buffer and metadata. */
+    describeImage: (imageBuffer: ArrayBuffer, meta: ExtractedImageMeta) => Promise<string>;
+    /** Optionally extract entities/relationships from all image descriptions. */
+    extractEntities?: (descriptions: string[]) => Promise<{
+        entities: IngestEntity$1[];
+        relationships: IngestRelationship$1[];
+    }>;
 }
-
 /** Error thrown by MemoryClient when the server returns a non-success response. */
 declare class MemoryServerError extends Error {
     readonly status: number;
@@ -95,9 +97,17 @@ declare class MemoryClient implements ExtendedMemoryInterface {
     stats(): Promise<MemoryStats$1>;
     shutdown(): Promise<void>;
     list(params?: MemoryListParams): Promise<MemoryListResult>;
+    /**
+     * Ingest a file with optional two-phase enrichment.
+     *
+     * Without enrichment callbacks: standard single-phase ingest (backwards compatible).
+     * With enrichment callbacks: fetches extracted images, calls describeImage for each,
+     * optionally extracts entities, then submits enrichment data back to the server.
+     */
     ingestFile(file: File, options?: {
         description?: string;
-    }): Promise<IngestionResult>;
+        enrichment?: EnrichmentCallbacks;
+    }): Promise<FileIngestResult>;
     /** @deprecated Use {@link list} instead. Kept for backwards compatibility. */
     listEntries(params?: {
         page?: number;
@@ -127,6 +137,14 @@ declare class MemoryClient implements ExtendedMemoryInterface {
     private parseApiResponse;
 }
 
+interface IngestionResult {
+    filename: string;
+    fileType: string;
+    chunks: number;
+    entryIds: string[];
+    totalCharacters: number;
+}
+
 declare const DEFAULTS: {
     readonly DATA_DIR: "./data";
     readonly VECTOR_PROVIDER: "lancedb";
@@ -336,4 +354,4 @@ interface GraphTraversalResult {
     }>;
 }
 
-export { type AgentId, type ApiResponse, type ConsolidationRunResult, DEFAULTS, EmbeddingProviderName, type ExtendedMemoryInterface, type GraphNode, type GraphRelationship, type GraphTraversalResult, type IngestEntity, type IngestRelationship, type IngestionResult, MemoryClient, type MemoryEntry, type MemoryIngestRequest, type MemoryInterface, type MemoryListParams, type MemoryListResult, type MemorySearchParams, type MemorySearchResult, MemoryServerError, type MemoryStats, MemoryType, RAGStrategy, type StoreInput, StoreTarget, type TemporalQueryFilters, type Timestamp, VectorProvider };
+export { type AgentId, type ApiResponse, type ConsolidationRunResult, DEFAULTS, EmbeddingProviderName, type EnrichmentCallbacks, type ExtendedMemoryInterface, type GraphNode, type GraphRelationship, type GraphTraversalResult, type IngestEntity, type IngestRelationship, type IngestionResult, MemoryClient, type MemoryEntry, type MemoryIngestRequest, type MemoryInterface, type MemoryListParams, type MemoryListResult, type MemorySearchParams, type MemorySearchResult, MemoryServerError, type MemoryStats, MemoryType, RAGStrategy, type StoreInput, StoreTarget, type TemporalQueryFilters, type Timestamp, VectorProvider };

package/dist/index.mjs

@@ -1,7 +1,7 @@
 import {
   MemoryClient,
   MemoryServerError
-} from "./chunk-FEOGTCSW.mjs";
+} from "./chunk-YL5QMN6G.mjs";
 
 // ../shared/src/constants/defaults.ts
 var DEFAULTS = {

package/dist/react.mjs

@@ -11,8 +11,8 @@ import {
   toGraphologyFormat,
   transformGraphData,
   unreachableHealth
-} from "./chunk-L62JXWDM.mjs";
-import "./chunk-FEOGTCSW.mjs";
+} from "./chunk-AXTWVLDO.mjs";
+import "./chunk-YL5QMN6G.mjs";
 
 // ../dashboard/src/hooks/use-consolidation-log.ts
 import { useCallback as useCallback2, useMemo } from "react";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pyxmate/memory",
-  "version": "0.3.0-beta",
+  "version": "0.4.2",
   "type": "module",
   "description": "SDK for pyx-memory — Memory as a Service for AI agents",
   "license": "MIT",

package/skills/pyx-memory/patterns/consumer.md

@@ -38,6 +38,15 @@ if (memory) {
   const file = new File([buffer], 'report.pdf', { type: 'application/pdf' });
   const result = await memory.ingestFile(file);
 
+  // Image ingestion with AI description
+  // pyx-memory saves the original image to {DATA_DIR}/files/ and indexes the
+  // description in vector + SQLite for semantic search. Without a description,
+  // images get a minimal placeholder ("[Image] filename (size KB)").
+  const image = new File([imgBuffer], 'photo.png', { type: 'image/png' });
+  const imgResult = await memory.ingestFile(image, {
+    description: 'A screenshot of the login page showing an error dialog',
+  });
+
   // Lifecycle
   await memory.consolidate();
   await memory.runDecay();

package/skills/pyx-memory/reference/http-api.md

@@ -2,7 +2,7 @@
 
 ## Authentication
 
-When the server has `API_KEY` configured, all requests (except `/health`) require one of:
+When the server has `API_KEY` configured, all requests (except `/health` and enrichment routes) require one of:
 
 ```
 Authorization: Bearer <your-api-key>
@@ -11,19 +11,23 @@ X-API-Key: <your-api-key>
 
 Destructive operations (DELETE, forget, decay, consolidate, reindex) require `ADMIN_API_KEY` when configured (falls back to `API_KEY`).
 
+**Enrichment routes** (`/api/memory/files/*`) use HMAC token auth instead of API keys. Tokens are issued by the server during file ingestion and are short-lived (30 minutes).
+
 `MemoryClient` handles this automatically when `apiKey` is passed to the constructor:
 ```typescript
 const client = new MemoryClient('http://localhost:7822', process.env.MEMORY_API_KEY);
 ```
 
-## Core (10 endpoints)
+## Core (12 endpoints)
 
 | Method | Endpoint | Description |
 |--------|----------|-------------|
 | GET | `/health` | Public health check (status only — no internals exposed) |
 | GET | `/admin/health` | Admin health check (version, uptime, embedding provider, memory stats) |
 | POST | `/api/memory/ingest` | Store a memory (JSON: `{ content, type, metadata, agentId?, sessionId?, targets?, entities?, relationships?, importance?, source?, eventTime?, id?, parentId?, ingestTime? }`) |
-| POST | `/api/memory/ingest/file` | Upload file (multipart, 50MB limit). Supports optional `description` field for agent-provided image descriptions (via LLM vision). Formats: txt, md, csv, pdf, docx, json, jsonl, html, png, jpg, jpeg, webp, gif, bmp, tiff, svg |
+| POST | `/api/memory/ingest/file` | Upload file (multipart, 50MB limit). For PDFs with images, returns an `enrichment` block with HMAC token and image metadata for two-phase enrichment. |
+| GET | `/api/memory/files/:fileId/images/:imageId?token=...` | Fetch an extracted PDF image (binary). Requires HMAC token from ingest response. |
+| POST | `/api/memory/files/:fileId/enrich` | Submit image descriptions + entities after LLM vision processing. Requires `X-Enrichment-Token` header. |
 | GET | `/api/memory/search?query=...&strategy=...&limit=...` | Search memories — **does NOT support** filters, enableHyDE, enableRerank |
 | GET | `/api/memory/stats` | Memory statistics |
 | GET | `/api/memory/entries?page=...&limit=...` | List entries (paginated) |
@@ -54,6 +58,129 @@ const client = new MemoryClient('http://localhost:7822', process.env.MEMORY_API_
 | GET | `/api/memory/query-as-of?asOf=...` | Bi-temporal point-in-time query (asOf, type, agentId, source, limit) |
 | GET | `/api/memory/query-by-event-time?startTime=...&endTime=...` | Bi-temporal event time range query (startTime, endTime, type, agentId, source, limit) |
 
+## File Ingestion (Images + Documents)
+
+`POST /api/memory/ingest/file` accepts multipart/form-data with:
+
+| Field | Type | Required | Description |
+|-------|------|----------|-------------|
+| `file` | File | Yes | The file to ingest (max 50MB) |
+| `description` | string | No | Agent-provided description (e.g., from LLM vision). Used instead of parser output for images. |
+
+**Supported formats**: `.txt`, `.md`, `.csv`, `.pdf`, `.docx`, `.json`, `.jsonl`, `.html`, `.htm`, `.log`, `.tsv`, `.png`, `.jpg`, `.jpeg`, `.webp`, `.gif`, `.bmp`, `.tiff`, `.svg`
+
+**What happens on upload**:
+1. Original file is saved to `{DATA_DIR}/files/{filename}` (persistent across restarts)
+2. Text is extracted (documents) or description is used (images)
+3. Content is chunked and stored in SQLite + vector for semantic search
+4. Source-aware dedup: re-uploading the same filename replaces the previous version
+5. **PDFs with images**: Images are extracted via pdfjs-dist, saved to a temp directory, and an `enrichment` block is returned with HMAC-signed tokens for two-phase enrichment
+
+**Image ingestion**: Images cannot have text extracted. Pass a `description` field with a natural-language description (e.g., from an LLM with vision). Without a description, images get a minimal placeholder (`[Image] filename (size KB)`).
+
+**PDF enrichment**: When a PDF contains images (≥50x50px), the response includes an `enrichment` block. The SDK's `ingestFile()` with `EnrichmentCallbacks` handles the full flow automatically — see [sdk-guide.md](sdk-guide.md#two-phase-pdf-enrichment).
+
+### Example: ingest a document
+
+```bash
+curl -X POST {{ENDPOINT}}/api/memory/ingest/file \
+  -H "Authorization: Bearer {{API_KEY}}" \
+  -F "file=@report.pdf"
+```
+
+### Example: ingest an image with description
+
+```bash
+curl -X POST {{ENDPOINT}}/api/memory/ingest/file \
+  -H "Authorization: Bearer {{API_KEY}}" \
+  -F "file=@screenshot.png" \
+  -F "description=A screenshot of the dashboard showing memory usage at 85%"
+```
+
+### SDK usage
+
+```typescript
+// Document (text-only)
+const doc = new File([buffer], 'report.pdf', { type: 'application/pdf' });
+await memory.ingestFile(doc);
+
+// Image with description
+const img = new File([imgBuffer], 'photo.png', { type: 'image/png' });
+await memory.ingestFile(img, {
+  description: 'A photo of the whiteboard from the architecture meeting',
+});
+
+// PDF with two-phase enrichment (images described via LLM vision)
+await memory.ingestFile(pdfFile, {
+  enrichment: {
+    describeImage: async (buffer, meta) => {
+      // Call your LLM vision model to describe the image
+      return 'A bar chart showing Q4 revenue growth of 23%';
+    },
+    extractEntities: async (descriptions) => ({
+      entities: [{ name: 'Q4 Revenue', type: 'METRIC' }],
+      relationships: [],
+    }),
+  },
+});
+```
+
+## Two-Phase PDF Enrichment
+
+When a PDF contains extractable images, the server returns an `enrichment` block from `POST /api/memory/ingest/file`. The SDK wraps the full three-phase flow into a single `ingestFile()` call with `EnrichmentCallbacks`.
+
+### Flow
+
+```
+Phase 1: POST /api/memory/ingest/file → text stored immediately
+  Response: { ...result, enrichment: { fileId, token, expiresAt, images: [...] } }
+
+Phase 2: GET /api/memory/files/{fileId}/images/{imageId}?token=... → binary image
+  (SDK fetches each image, calls describeImage callback)
+
+Phase 3: POST /api/memory/files/{fileId}/enrich
+  Header: X-Enrichment-Token: {token}:{expiresAt}
+  Body: { imageDescriptions: [...], entities?: [...], relationships?: [...] }
+  → descriptions stored as memory entries, temp images cleaned up
+```
+
+### Image fetch endpoint
+
+`GET /api/memory/files/:fileId/images/:imageId?token=<hmac>`
+
+Returns the binary image with `Content-Type: image/png` or `image/jpeg`. The HMAC token is issued by the server during Phase 1 and expires after 30 minutes.
+
+### Enrich endpoint
+
+`POST /api/memory/files/:fileId/enrich`
+
+| Header | Required | Description |
+|--------|----------|-------------|
+| `X-Enrichment-Token` | Yes | Format: `{hmac_token}:{expiresAt_iso}` |
+
+Body:
+```json
+{
+  "imageDescriptions": [
+    { "imageId": "uuid-img-0", "description": "A chart showing..." }
+  ],
+  "entities": [{ "name": "Revenue", "type": "METRIC" }],
+  "relationships": []
+}
+```
+
+Input limits: descriptions max 10,000 chars each, entities max 200, relationships max 500.
+
+### Security
+
+- Tokens are HMAC-SHA256 signed (server secret + fileId + expiresAt)
+- `fileId` must be a valid UUIDv4
+- `imageId` must match `^[a-zA-Z0-9_-]+$`
+- Image paths are sandboxed via `resolve()` + `startsWith()` to prevent path traversal
+- Enrichment routes bypass API key auth — they use their own HMAC verification
+
+---
+
 ## Entity Extraction (Knowledge Graph)
 
 When storing memories that mention named subjects, include `entities` and `relationships` in the ingest request to populate the knowledge graph. This enables graph-based RAG and dashboard visualization.
@@ -121,3 +248,4 @@ All responses follow: `{ success: boolean, data?: T, error?: string }`
 | `NODE_ENV` | `development` | Set to `production` to mask 5xx error details and enable HSTS |
 | `PII_POLICY` | `flag` | PII handling: `flag` (detect + tag), `redact` (replace with [REDACTED]), `block` (reject 400) |
 | `RATE_LIMIT_RPM` | `0` | Requests per minute per IP. 0 = disabled |
+| `ENRICHMENT_SECRET` | (auto-generated) | HMAC secret for enrichment token signing. Auto-generated if unset (tokens won't survive restarts). Min 32 bytes for production. |

package/skills/pyx-memory/reference/sdk-guide.md

@@ -97,7 +97,7 @@ Start the server: `bun packages/server/src/index.ts`
 @pyx-memory/core     → Memory class, embeddings, graph, RAG, ingestion, lifecycle
        ↑                (re-exports everything from client and shared)
        |
-@pyx-memory/server   → HTTP sidecar server (23 endpoints)
+@pyx-memory/server   → HTTP sidecar server (25 endpoints)
 
 @pyx-memory/dashboard → DashboardClient (extends MemoryClient), React hooks,
                          Poller, aggregations, graph transforms (D3/Graphology)
@@ -164,3 +164,39 @@ MemoryServerError           — HTTP client errors (has .status, .isNotFound)
 ```
 
 All from `@pyx-memory/core` except `MemoryServerError` from `@pyx-memory/client`.
+
+## Two-Phase PDF Enrichment
+
+When ingesting PDFs that contain images, `ingestFile()` supports a two-phase enrichment flow via `EnrichmentCallbacks`. The SDK handles all three phases automatically:
+
+1. **Upload** — text extracted and stored immediately
+2. **Image description** — each extracted image is fetched and described via your LLM vision callback (batched, 5 concurrent)
+3. **Enrichment** — descriptions + entities submitted back to the server
+
+```typescript
+import { MemoryClient, type EnrichmentCallbacks } from '@pyx-memory/client';
+
+const memory = new MemoryClient('http://localhost:7822', apiKey);
+
+const enrichment: EnrichmentCallbacks = {
+  // Required: describe each image using LLM vision
+  describeImage: async (imageBuffer, meta) => {
+    // imageBuffer is ArrayBuffer of the extracted PNG/JPEG
+    // meta has: imageId, pageNumber, width, height, mimeType
+    const description = await myVisionModel.describe(imageBuffer);
+    return description;
+  },
+  // Optional: extract entities from all descriptions at once
+  extractEntities: async (descriptions) => ({
+    entities: [{ name: 'Revenue', type: 'METRIC' }],
+    relationships: [{ source: 'Revenue', target: 'Q4', type: 'RELATED_TO' }],
+  }),
+};
+
+const result = await memory.ingestFile(pdfFile, { enrichment });
+// result.enrichment is defined when PDF had images
+// Text chunks are already stored from Phase 1
+// Image descriptions are stored from Phase 3
+```
+
+Without `enrichment` callbacks, `ingestFile()` behaves identically to before — fully backwards compatible.

package/skills/pyx-memory/reference/types.md

@@ -28,6 +28,13 @@
 | `MemoryListParams` | `{ page?, limit?, type?, agentId? }` | `@pyx-memory/client` |
 | `MemoryListResult` | `{ entries, totalCount, page, limit }` | `@pyx-memory/client` |
 | `IngestionResult` | `{ filename, chunks, totalCharacters }` | `@pyx-memory/client` |
+| `FileIngestResult` | `IngestionResult & { enrichment?: EnrichmentPending }` | `@pyx-memory/shared` |
+| `EnrichmentPending` | `{ fileId, token, expiresAt, images: ExtractedImageMeta[] }` | `@pyx-memory/shared` |
+| `ExtractedImageMeta` | `{ imageId, pageNumber, width, height, mimeType }` | `@pyx-memory/shared` |
+| `EnrichmentCallbacks` | `{ describeImage, extractEntities? }` | `@pyx-memory/client` |
+| `EnrichRequest` | `{ imageDescriptions, entities?, relationships? }` | `@pyx-memory/shared` |
+| `EnrichResult` | `{ entryIds, entitiesStored, relationshipsStored }` | `@pyx-memory/shared` |
+| `ImageDescription` | `{ imageId, description }` | `@pyx-memory/shared` |
 | `MemoryServerError` | `Error` with `.status` and `.isNotFound` | `@pyx-memory/client` |
 | `GraphNode` | `{ id, name, type, properties, memoryEntryIds }` | `@pyx-memory/shared` |
 | `GraphTraversalResult` | `{ nodes, relationships, paths }` | `@pyx-memory/shared` |
@@ -86,8 +93,13 @@ class MemoryClient implements ExtendedMemoryInterface {
   graphEdges(): Promise<{ stats: { nodeCount: number; edgeCount: number } }>;
   graphQuery(query: { nodeId: string; depth?: number }): Promise<GraphTraversalResult>;
 
-  // File ingestion (multipart upload to server)
-  ingestFile(file: File): Promise<IngestionResult>;
+  // File ingestion with optional two-phase enrichment
+  // Without callbacks: standard single-phase ingest (backwards compatible)
+  // With callbacks: fetches extracted images, calls describeImage, submits enrichment
+  ingestFile(file: File, options?: {
+    description?: string;
+    enrichment?: EnrichmentCallbacks;
+  }): Promise<FileIngestResult>;
 
   // Bi-temporal queries
   queryAsOf(asOf: string, filters?: TemporalQueryFilters): Promise<MemoryEntry[]>;