latticesql 0.7.0 → 0.9.0

package/README.md

@@ -356,13 +356,16 @@ db.defineMulti('agent-context', {
 db.defineEntityContext(table: string, def: EntityContextDefinition): this
 ```
 
-Generate a **parallel file-system tree** for an entity type — one subdirectory per row, one file per declared relationship, and an optional combined context file. Must be called before `init()`.
+Generate a **parallel file-system tree** for an entity type — one subdirectory per row, one file per declared relationship, and an optional combined context file. Can be called before or after `init()`.
 
 ```typescript
 db.defineEntityContext('agents', {
   // Derive the subdirectory name for each entity
   slug: (row) => row.slug as string,
 
+  // Default query options for all relationship sources (v0.6+)
+  sourceDefaults: { softDelete: true },
+
   // Global index file listing all entities
   index: {
     outputFile: 'agents/AGENTS.md',
@@ -376,7 +379,8 @@ db.defineEntityContext('agents', {
       render: ([r]) => `# ${r.name as string}\n\n${r.bio as string ?? ''}`,
     },
     'TASKS.md': {
-      source: { type: 'hasMany', table: 'tasks', foreignKey: 'agent_id' },
+      source: { type: 'hasMany', table: 'tasks', foreignKey: 'agent_id',
+                orderBy: 'created_at', orderDir: 'desc', limit: 20 },
       render: (rows) => rows.map((r) => `- ${r.title as string}`).join('\n'),
       omitIfEmpty: true,                               // skip if no tasks
       budget: 4000,                                    // truncate at 4 000 chars
@@ -388,6 +392,7 @@ db.defineEntityContext('agents', {
         localKey: 'agent_id',
         remoteKey: 'skill_id',
         remoteTable: 'skills',
+        orderBy: 'name',                               // softDelete inherited from sourceDefaults
       },
       render: (rows) => rows.map((r) => `- ${r.name as string}`).join('\n'),
       omitIfEmpty: true,
@@ -423,11 +428,82 @@ context/
 | Type | What it queries |
 |---|---|
 | `{ type: 'self' }` | The entity row itself |
-| `{ type: 'hasMany', table, foreignKey, references? }` | Rows in `table` where `foreignKey = entityPk` |
-| `{ type: 'manyToMany', junctionTable, localKey, remoteKey, remoteTable, references? }` | Remote rows via a junction table |
-| `{ type: 'belongsTo', table, foreignKey, references? }` | Single parent row via FK on this entity (`null` FK → empty) |
+| `{ type: 'hasMany', table, foreignKey, ... }` | Rows in `table` where `foreignKey = entityPk` |
+| `{ type: 'manyToMany', junctionTable, localKey, remoteKey, remoteTable, ... }` | Remote rows via a junction table |
+| `{ type: 'belongsTo', table, foreignKey, ... }` | Single parent row via FK on this entity (`null` FK → empty) |
+| `{ type: 'enriched', include: { ... } }` | Entity row + related data attached as `_key` JSON fields (v0.7+) |
 | `{ type: 'custom', query: (row, adapter) => Row[] }` | Fully custom synchronous query |
 
+#### Source query options (v0.6+)
+
+`hasMany`, `manyToMany`, and `belongsTo` sources accept optional query refinements:
+
+```typescript
+{
+  type: 'hasMany',
+  table: 'tasks',
+  foreignKey: 'agent_id',
+  // Query options (all optional):
+  softDelete: true,          // exclude rows where deleted_at IS NULL
+  filters: [                 // additional WHERE clauses (uses existing Filter type)
+    { col: 'status', op: 'eq', val: 'active' },
+  ],
+  orderBy: 'created_at',    // ORDER BY column
+  orderDir: 'desc',         // 'asc' (default) or 'desc'
+  limit: 20,                // LIMIT N
+}
+```
+
+The `softDelete: true` shorthand is equivalent to `filters: [{ col: 'deleted_at', op: 'isNull' }]`.
+
+#### sourceDefaults (v0.6+)
+
+Set default query options for all relationship sources in an entity context:
+
+```typescript
+db.defineEntityContext('agents', {
+  slug: (row) => row.slug as string,
+  sourceDefaults: { softDelete: true },  // applied to all hasMany/manyToMany/belongsTo
+  files: {
+    'TASKS.md': {
+      // softDelete: true is inherited from sourceDefaults
+      source: { type: 'hasMany', table: 'tasks', foreignKey: 'agent_id', orderBy: 'created_at' },
+      render: (rows) => rows.map((r) => `- ${r.title as string}`).join('\n'),
+    },
+  },
+});
+```
+
+Per-file source options override defaults. `custom`, `self`, and `enriched` sources are unaffected.
+
+#### Enriched source (v0.7+)
+
+Starts with the entity's own row and attaches related data as JSON string fields. Each key in `include` becomes a `_key` field containing `JSON.stringify(resolvedRows)`.
+
+```typescript
+'PROFILE.md': {
+  source: {
+    type: 'enriched',
+    include: {
+      // Declarative sub-lookups (support all query options)
+      skills:   { type: 'manyToMany', junctionTable: 'agent_skills',
+                  localKey: 'agent_id', remoteKey: 'skill_id',
+                  remoteTable: 'skills', softDelete: true },
+      projects: { type: 'hasMany', table: 'projects', foreignKey: 'org_id',
+                  softDelete: true, orderBy: 'name' },
+      // Custom sub-lookup for complex queries
+      stats:    { type: 'custom', query: (row, adapter) =>
+                    adapter.all('SELECT COUNT(*) as cnt FROM events WHERE actor_id = ?', [row.id]) },
+    },
+  },
+  render: ([row]) => {
+    const skills = JSON.parse(row._skills as string);
+    const projects = JSON.parse(row._projects as string);
+    return `# ${row.name}\n\nSkills: ${skills.length}\nProjects: ${projects.length}`;
+  },
+}
+```
+
 See [docs/entity-context.md](./docs/entity-context.md) for the complete guide.
 
 ---
@@ -924,6 +1000,71 @@ db.define('tickets', {
 
 ---
 
+## Markdown utilities (v0.6+)
+
+Composable helper functions for building render functions. Use inside `render: (rows) => ...` callbacks to reduce boilerplate.
+
+### `frontmatter(fields)`
+
+Generate a YAML-style frontmatter block. Automatically includes `generated_at` with the current ISO timestamp.
+
+```typescript
+import { frontmatter } from 'latticesql';
+
+const header = frontmatter({ agent: 'Alice', skill_count: 5 });
+// ---
+// generated_at: "2026-03-27T..."
+// agent: "Alice"
+// skill_count: 5
+// ---
+```
+
+### `markdownTable(rows, columns)`
+
+Generate a GitHub-Flavoured Markdown table from rows with explicit column configuration and optional per-cell formatters.
+
+```typescript
+import { markdownTable } from 'latticesql';
+
+const md = markdownTable(rows, [
+  { key: 'name',   header: 'Name' },
+  { key: 'status', header: 'Status', format: (v) => String(v || '—') },
+  { key: 'name',   header: 'Detail', format: (v, row) => `[view](${row.slug}/DETAIL.md)` },
+]);
+// | Name | Status | Detail |
+// | --- | --- | --- |
+// | Alice | active | [view](alice/DETAIL.md) |
+```
+
+Returns empty string for zero rows. The `format` callback receives `(cellValue, fullRow)`.
+
+### `slugify(name)`
+
+Generate a URL-safe slug from a display name — lowercases, strips diacritics, replaces non-alphanumeric runs with hyphens.
+
+```typescript
+import { slugify } from 'latticesql';
+
+slugify('My Agent Name');  // 'my-agent-name'
+slugify('Jose Garcia');    // 'jose-garcia'
+```
+
+### `truncate(content, maxChars, notice?)`
+
+Truncate content at a character budget. Appends a notice when truncation occurs.
+
+```typescript
+import { truncate } from 'latticesql';
+
+const md = truncate(longContent, 4000);
+// Appends: "\n\n*[truncated — context budget exceeded]*"
+
+const md2 = truncate(longContent, 4000, '\n\n[...truncated]');
+// Custom notice
+```
+
+---
+
 ## Entity context directories (v0.5+)
 
 `defineEntityContext()` is the high-level API for per-entity file generation — the pattern where each entity type gets its own directory tree, with a separate file for each relationship type.
@@ -1071,6 +1212,74 @@ interface SessionWriteEntry {
 
 The processor is responsible for applying the parsed entries to your DB and validating field names against your schema. The `parseSessionWrites` function is pure — no DB access, no side effects.
 
+### Full session parser (v0.5.2+)
+
+For parsing **all** entry types (not just writes), use `parseSessionMD`:
+
+```ts
+import { parseSessionMD, parseMarkdownEntries } from 'latticesql';
+
+// Parse YAML-delimited entries (--- header --- body ===)
+const result = parseSessionMD(content, startOffset);
+// result.entries: SessionEntry[]  — all types: event, learning, status, write, etc.
+// result.errors:  ParseError[]
+// result.lastOffset: number       — for incremental parsing
+
+// Parse markdown heading entries (## timestamp — description)
+const mdResult = parseMarkdownEntries(content, 'agent-name', startOffset);
+```
+
+#### Configurable entry types (v0.5.5+)
+
+By default, the parser validates against a built-in set of entry types. Override via `SessionParseOptions`:
+
+```ts
+import { parseSessionMD, DEFAULT_ENTRY_TYPES, DEFAULT_TYPE_ALIASES } from 'latticesql';
+
+// Accept any type (no validation)
+parseSessionMD(content, 0, { validTypes: null });
+
+// Custom type set
+parseSessionMD(content, 0, {
+  validTypes: new Set(['alert', 'todo', 'write']),
+  typeAliases: { warning: 'alert', task: 'todo' },
+});
+```
+
+### Read-only header (v0.5.5+)
+
+All generated context files should carry a read-only header. Use the default or create a custom one:
+
+```ts
+import { READ_ONLY_HEADER, createReadOnlyHeader } from 'latticesql';
+
+// Default: "generated by Lattice"
+const header = READ_ONLY_HEADER;
+
+// Custom generator name and docs reference
+const custom = createReadOnlyHeader({
+  generator: 'my-sync-tool',
+  docsRef: 'https://example.com/docs/sessions',
+});
+```
+
+### Write applicator (v0.5.2+)
+
+Apply parsed write entries to a better-sqlite3 database with schema validation:
+
+```ts
+import { applyWriteEntry } from 'latticesql';
+
+const result = applyWriteEntry(db, writeEntry);
+if (result.ok) {
+  console.log(`Applied to ${result.table}, record ${result.recordId}`);
+} else {
+  console.error(result.reason);
+}
+```
+
+Validates table existence, field names against schema, and uses soft-delete when a `deleted_at` column exists.
+
 ---
 
 ## YAML config (v0.4+)

package/dist/cli.js

@@ -666,9 +666,18 @@ function appendQueryOptions(baseSql, params, opts, tableAlias) {
         break;
     }
   }
-  if (opts.orderBy && SAFE_COL_RE.test(opts.orderBy)) {
-    const dir = opts.orderDir === "desc" ? "DESC" : "ASC";
-    sql += ` ORDER BY ${prefix}"${opts.orderBy}" ${dir}`;
+  if (opts.orderBy) {
+    if (typeof opts.orderBy === "string") {
+      if (SAFE_COL_RE.test(opts.orderBy)) {
+        const dir = opts.orderDir === "desc" ? "DESC" : "ASC";
+        sql += ` ORDER BY ${prefix}"${opts.orderBy}" ${dir}`;
+      }
+    } else {
+      const clauses = opts.orderBy.filter((spec) => SAFE_COL_RE.test(spec.col)).map((spec) => `${prefix}"${spec.col}" ${spec.dir === "desc" ? "DESC" : "ASC"}`);
+      if (clauses.length > 0) {
+        sql += ` ORDER BY ${clauses.join(", ")}`;
+      }
+    }
   }
   if (opts.limit !== void 0 && opts.limit > 0) {
     sql += ` LIMIT ${Math.floor(opts.limit)}`;
@@ -691,7 +700,19 @@ function resolveEntitySource(source, entityRow, entityPk, adapter) {
       const pkVal = entityRow[entityPk];
       const remotePk = source.references ?? "id";
       const params = [pkVal];
-      let sql = `SELECT r.* FROM "${source.remoteTable}" r
+      let selectCols = "r.*";
+      if (source.junctionColumns?.length) {
+        const jCols = source.junctionColumns.map((jc) => {
+          if (typeof jc === "string") {
+            if (!SAFE_COL_RE.test(jc)) return null;
+            return `j."${jc}"`;
+          }
+          if (!SAFE_COL_RE.test(jc.col) || !SAFE_COL_RE.test(jc.as)) return null;
+          return `j."${jc.col}" AS "${jc.as}"`;
+        }).filter(Boolean);
+        if (jCols.length > 0) selectCols += ", " + jCols.join(", ");
+      }
+      let sql = `SELECT ${selectCols} FROM "${source.remoteTable}" r
          JOIN "${source.junctionTable}" j ON j."${source.remoteKey}" = r."${remotePk}"
          WHERE j."${source.localKey}" = ?`;
       sql = appendQueryOptions(sql, params, source, "r");
@@ -736,6 +757,165 @@ function truncateContent(content, budget) {
   return content.slice(0, budget) + "\n\n*[truncated \u2014 context budget exceeded]*";
 }
 
+// src/render/markdown.ts
+function frontmatter(fields) {
+  const lines = [`generated_at: "${(/* @__PURE__ */ new Date()).toISOString()}"`];
+  for (const [key, val] of Object.entries(fields)) {
+    lines.push(typeof val === "string" ? `${key}: "${val}"` : `${key}: ${String(val)}`);
+  }
+  return `---
+${lines.join("\n")}
+---
+
+`;
+}
+function markdownTable(rows, columns) {
+  if (rows.length === 0 || columns.length === 0) return "";
+  const header = "| " + columns.map((c) => c.header).join(" | ") + " |";
+  const separator = "| " + columns.map(() => "---").join(" | ") + " |";
+  const body = rows.map((row) => {
+    const cells = columns.map((col) => {
+      const raw = row[col.key];
+      return col.format ? col.format(raw, row) : String(raw ?? "");
+    });
+    return "| " + cells.join(" | ") + " |";
+  });
+  return [header, separator, ...body].join("\n") + "\n";
+}
+
+// src/session/constants.ts
+function createReadOnlyHeader(options) {
+  const generator = options?.generator ?? "Lattice";
+  const docsRef = options?.docsRef ?? "the Lattice documentation";
+  return `<!-- READ ONLY \u2014 generated by ${generator}. Do not edit directly.
+     To update data in Lattice: write entries to SESSION.md in this directory.
+     Format: type: write | op: create/update/delete | table: <name> | target: <id>
+     See ${docsRef} for the SESSION.md format spec. -->
+
+`;
+}
+var READ_ONLY_HEADER = createReadOnlyHeader();
+
+// src/render/entity-templates.ts
+var DEFAULT_HEADER = createReadOnlyHeader();
+function compileEntityRender(spec) {
+  if (typeof spec === "function") return spec;
+  return compileTemplate(spec);
+}
+function compileTemplate(tmpl) {
+  switch (tmpl.template) {
+    case "entity-table":
+      return compileEntityTable(tmpl);
+    case "entity-profile":
+      return compileEntityProfile(tmpl);
+    case "entity-sections":
+      return compileEntitySections(tmpl);
+  }
+}
+function compileEntityTable(tmpl) {
+  return (rows) => {
+    const data = tmpl.beforeRender ? tmpl.beforeRender(rows) : rows;
+    let md = DEFAULT_HEADER;
+    md += frontmatter(tmpl.frontmatter ?? {});
+    md += `# ${tmpl.heading}
+
+`;
+    if (data.length === 0) {
+      md += tmpl.emptyMessage ?? "*No data.*\n";
+    } else {
+      md += markdownTable(data, tmpl.columns);
+    }
+    return md;
+  };
+}
+function compileEntityProfile(tmpl) {
+  return (rows) => {
+    const data = tmpl.beforeRender ? tmpl.beforeRender(rows) : rows;
+    const r = data[0];
+    if (!r) return "";
+    let md = DEFAULT_HEADER;
+    if (tmpl.frontmatter) {
+      const fm = typeof tmpl.frontmatter === "function" ? tmpl.frontmatter(r) : tmpl.frontmatter;
+      md += frontmatter(fm);
+    } else {
+      md += frontmatter({});
+    }
+    const heading = typeof tmpl.heading === "function" ? tmpl.heading(r) : tmpl.heading;
+    md += `# ${heading}
+
+`;
+    for (const field of tmpl.fields) {
+      const val = r[field.key];
+      if (val === null || val === void 0) continue;
+      const formatted = field.format ? field.format(val, r) : String(val);
+      if (formatted) {
+        md += `**${field.label}:** ${formatted}
+`;
+      }
+    }
+    if (tmpl.sections) {
+      for (const section of tmpl.sections) {
+        const rawJson = r[`_${section.key}`];
+        if (!rawJson) continue;
+        if (section.condition && !section.condition(r)) continue;
+        const items = JSON.parse(rawJson);
+        if (items.length === 0) continue;
+        const sectionHeading = typeof section.heading === "function" ? section.heading(r) : section.heading;
+        md += `
+## ${sectionHeading}
+
+`;
+        if (section.render === "table" && section.columns) {
+          md += markdownTable(items, section.columns);
+        } else if (section.render === "list" && section.formatItem) {
+          for (const item of items) {
+            md += `- ${section.formatItem(item)}
+`;
+          }
+        } else if (typeof section.render === "function") {
+          md += section.render(items);
+        }
+      }
+    }
+    return md;
+  };
+}
+function compileEntitySections(tmpl) {
+  return (rows) => {
+    const data = tmpl.beforeRender ? tmpl.beforeRender(rows) : rows;
+    let md = DEFAULT_HEADER;
+    md += frontmatter(tmpl.frontmatter ?? {});
+    md += `# ${tmpl.heading}
+
+`;
+    if (data.length === 0) {
+      md += tmpl.emptyMessage ?? "*No data.*\n";
+      return md;
+    }
+    for (const row of data) {
+      md += `## ${tmpl.perRow.heading(row)}
+`;
+      if (tmpl.perRow.metadata?.length) {
+        const parts = tmpl.perRow.metadata.map((m) => {
+          const val = row[m.key];
+          const formatted = m.format ? m.format(val) : String(val ?? "");
+          return `**${m.label}:** ${formatted}`;
+        }).filter(Boolean);
+        if (parts.length > 0) {
+          md += parts.join(" | ") + "\n";
+        }
+      }
+      if (tmpl.perRow.body) {
+        md += `
+${tmpl.perRow.body(row)}
+`;
+      }
+      md += "\n";
+    }
+    return md;
+  };
+}
+
 // src/lifecycle/cleanup.ts
 import { join as join4 } from "path";
 import { existsSync as existsSync4, readdirSync, unlinkSync, rmdirSync, statSync } from "fs";
@@ -945,7 +1125,8 @@ var RenderEngine = class {
           const source = mergeDefaults ? { ...def.sourceDefaults, ...spec.source } : spec.source;
           const rows = resolveEntitySource(source, entityRow, entityPk, this._adapter);
           if (spec.omitIfEmpty && rows.length === 0) continue;
-          const content = truncateContent(spec.render(rows), spec.budget);
+          const renderFn = compileEntityRender(spec.render);
+          const content = truncateContent(renderFn(rows), spec.budget);
           renderedFiles.set(filename, content);
           const filePath = join5(entityDir, filename);
           if (atomicWrite(filePath, content)) {