contentbase 0.0.2 → 0.0.5

package/README.md

@@ -5,8 +5,7 @@
 Contentbase treats a folder of Markdown and MDX files as a typed, queryable database. Define models with Zod schemas, extract structured data from headings and lists, traverse parent/child relationships across documents, validate everything, and query it all with a fluent API.
 
 ```ts
-import { Collection, defineModel, section, hasMany, z } from "contentbase";
-import { toString } from "mdast-util-to-string";
+import { Collection, defineModel, section, hasMany, z, toString } from "contentbase";
 
 const Story = defineModel("Story", {
   meta: z.object({
@@ -79,14 +78,18 @@ content/
 
 A model is a config object that describes one type of document. It declares:
 
+- **description** -- human-readable summary (auto-generated from schema if omitted)
 - **meta** -- a Zod schema for frontmatter
 - **sections** -- named extractions from heading-based sections
 - **relationships** -- `hasMany` / `belongsTo` links between models
 - **computed** -- derived values calculated from instance data
+- **defaults** -- static default values for frontmatter fields
+- **pattern** -- Express-style path patterns for inferring meta from file paths
 
 ```ts
 const Epic = defineModel("Epic", {
   prefix: "epics",
+  description: "A project epic that groups related user stories.",
   meta: z.object({
     priority: z.enum(["low", "medium", "high"]).optional(),
     status: z.enum(["created", "in-progress", "complete"]).default("created"),
@@ -103,14 +106,71 @@ const Epic = defineModel("Epic", {
 });
 ```
 
+If `description` is omitted, one is generated on first access from the model's schema: *"An Epic has metadata (priority, status), relationship (stories → Story), and computed property (isComplete)."*
+
 The `prefix` determines which files match this model. Files whose path starts with `"epics"` are Epics. If omitted, the prefix is auto-pluralized from the name (`"Epic"` -> `"epics"`).
 
+#### Explicit Model Assignment with `_model`
+
+Documents at the root of a collection (not in a subfolder) can't be matched by prefix. You can explicitly assign a model by adding `_model` to the frontmatter:
+
+```yaml
+---
+_model: Epic
+title: Platform Migration
+status: created
+---
+```
+
+The `_model` key takes priority over prefix matching, so this works even if the file lives outside the model's prefix folder.
+
+#### The Base Model
+
+Every collection automatically registers a built-in `Base` model as a catch-all. Documents that don't match any other model (by `_model` or prefix) are assigned to Base. You can query these unmatched documents:
+
+```ts
+import { Base } from "contentbase";
+
+const misc = await collection.query(Base).fetchAll();
+```
+
+If you want to customize the Base model (e.g. add a meta schema), register your own before calling `load()`:
+
+```ts
+const MyBase = defineModel("Base", {
+  meta: z.object({ tags: z.array(z.string()).optional() }),
+});
+collection.register(MyBase);
+await collection.load(); // won't auto-register the built-in Base
+```
+
+#### Path Patterns
+
+Models can declare Express-style path patterns to automatically infer meta values from the document's file path:
+
+```ts
+const Story = defineModel("Story", {
+  prefix: "stories",
+  pattern: "stories/:epic/:slug",
+  meta: z.object({
+    epic: z.string(),
+    slug: z.string(),
+  }),
+});
+```
+
+A file at `stories/authentication/user-can-register.mdx` will automatically have `{ epic: "authentication", slug: "user-can-register" }` inferred into its meta. Explicit frontmatter values always take precedence over pattern-inferred values. You can also supply an array of patterns -- the first match wins.
+
 ### Collections
 
 A `Collection` loads a directory tree and gives you access to documents and typed model instances.
 
 ```ts
-const collection = new Collection({ rootPath: "./content" });
+const collection = new Collection({
+  rootPath: "./content",
+  extensions: ["mdx", "md"],   // default
+  autoDiscover: true,          // auto-load models.ts if no models registered
+});
 await collection.load();
 
 // Register models for prefix-based matching
@@ -141,8 +201,7 @@ Given this Markdown:
 Define a section to extract the list items:
 
 ```ts
-import { section } from "contentbase";
-import { toString } from "mdast-util-to-string";
+import { section, toString } from "contentbase";
 
 const Story = defineModel("Story", {
   sections: {
@@ -150,12 +209,13 @@ const Story = defineModel("Story", {
       extract: (query) =>
         query.selectAll("listItem").map((node) => toString(node)),
       schema: z.array(z.string()),
+      alternatives: ["Requirements"],  // fallback heading names
     }),
   },
 });
 ```
 
-The `extract` function receives an `AstQuery` scoped to the content under that heading. The `schema` is optional and used during validation.
+The `extract` function receives an `AstQuery` scoped to the content under that heading. The `schema` is optional and used during validation. The `alternatives` array provides fallback heading names -- if "Acceptance Criteria" isn't found, it tries "Requirements" next.
 
 Section data is **lazily computed and cached** -- the extract function only runs the first time you access the property.
 
@@ -272,6 +332,9 @@ const results = await collection
   .query(Story)
   .whereIn("meta.status", ["created", "in-progress"])
   .whereExists("meta.epic")
+  .sort("meta.status", "asc")
+  .limit(10)
+  .offset(0)
   .fetchAll();
 
 // Convenience accessors
@@ -283,6 +346,60 @@ Available operators: `eq`, `neq`, `in`, `notIn`, `gt`, `lt`, `gte`, `lte`, `cont
 
 Queries filter by model type **before** creating instances, so you only pay the parsing cost for matching documents.
 
+### JSON Query DSL
+
+For querying over the wire (REST API, MCP server) without executing arbitrary code, Contentbase provides a MongoDB-style JSON query DSL:
+
+```json
+{
+  "model": "Epic",
+  "where": {
+    "meta.status": "created",
+    "meta.priority": { "$gt": 3 }
+  },
+  "sort": { "meta.priority": "desc" },
+  "select": ["id", "title", "meta.status"],
+  "limit": 10,
+  "offset": 0
+}
+```
+
+**Where value shortcuts:**
+
+| Value type | Interpretation | Example |
+|------------|---------------|---------|
+| Literal (string, number, boolean, null) | Implicit `$eq` | `"meta.status": "active"` |
+| Array | Implicit `$in` | `"meta.tags": ["a", "b"]` |
+| Operator object | Explicit operator | `"meta.priority": { "$gt": 5 }` |
+| Multiple operators | AND on same field | `"meta.priority": { "$gte": 3, "$lte": 8 }` |
+
+**Available operators:** `$eq`, `$neq`, `$in`, `$notIn`, `$gt`, `$lt`, `$gte`, `$lte`, `$contains`, `$startsWith`, `$endsWith`, `$regex`, `$exists`
+
+**Options:**
+
+| Field | Type | Default | Description |
+|-------|------|---------|-------------|
+| `model` | string | *(required)* | Model name or prefix |
+| `where` | object | -- | MongoDB-style filter conditions |
+| `sort` | object or array | -- | `{ "field": "asc"\|"desc" }` or `[{ "path": "...", "direction": "asc" }]` |
+| `select` | string[] | -- | Fields to include in output |
+| `limit` | number | -- | Maximum results to return |
+| `offset` | number | -- | Number of results to skip |
+| `method` | string | `"fetchAll"` | `"fetchAll"`, `"first"`, `"last"`, or `"count"` |
+
+Use via `POST /api/query` with a JSON body, or through the MCP `query` tool. The DSL parser is also available programmatically:
+
+```ts
+import { executeQueryDSL, queryDSLSchema } from "contentbase";
+
+const result = await executeQueryDSL(collection, {
+  model: "Epic",
+  where: { "meta.status": "created" },
+  sort: { title: "asc" },
+  limit: 5,
+});
+```
+
 ---
 
 ## Validation
@@ -489,6 +606,32 @@ collection.tableOfContents({ basePath: "./content" });
 // links become: ./content/epics/authentication.mdx
 ```
 
+### File Tree
+
+Render an ASCII file tree of all documents in the collection:
+
+```ts
+const tree = collection.renderFileTree();
+```
+
+```
+epics/
+├── authentication.mdx
+└── searching-and-browsing.mdx
+stories/
+└── authentication/
+    └── a-user-should-be-able-to-register.mdx
+```
+
+### Model Summary
+
+Generate comprehensive documentation of all registered models, including schema fields, sections, relationships, and defaults:
+
+```ts
+const summary = await collection.generateModelSummary();
+// Returns markdown documenting each model's schema, sections, relationships
+```
+
 ---
 
 ## Computed Properties
@@ -537,16 +680,141 @@ collection.use(timestampPlugin, { format: "iso" });
 
 ## CLI
 
-Contentbase ships with a CLI for common operations:
+Contentbase ships with a CLI available as both `cnotes` and `contentbase`. See [CLI.md](./CLI.md) for the full reference with examples for every command.
+
+```bash
+bun add contentbase
+
+# Then use it via bunx, or in package.json scripts
+bunx cnotes inspect
+```
+
+### Commands
+
+```bash
+cnotes init [name]                             # scaffold a new project
+cnotes create <Model> --title "..."            # scaffold a new document (uses templates if available)
+cnotes inspect                                 # show models, sections, relationships, doc counts
+cnotes validate [target]                       # validate documents ('all', a model name, or a path ID)
+cnotes export                                  # export collection as JSON
+cnotes extract <glob> --sections "A, B"        # extract specific sections from matching documents
+cnotes summary                                 # generate MODELS.md and TABLE-OF-CONTENTS.md
+cnotes teach                                   # output combined documentation for LLM context
+cnotes action <name>                           # run a named action
+cnotes text-search <pattern>                   # search file contents with pattern matching
+cnotes serve                                   # start HTTP server with REST API and doc serving
+cnotes mcp                                     # start MCP server for AI agent integration
+cnotes console                                 # interactive REPL with collection in scope
+cnotes help                                    # list available commands
+```
+
+All commands accept `--contentFolder` to specify which folder contains your content. Defaults to `./docs`. You can also set it in `package.json`:
+
+```json
+{
+  "contentbase": {
+    "contentFolder": "content"
+  }
+}
+```
+
+### serve
+
+Start an HTTP server that exposes a full REST API for the collection. Documents are available as JSON, rendered HTML, or raw markdown.
+
+```bash
+# Start on default port 8000
+cnotes serve
+
+# Custom port, specific content folder
+cnotes serve --port 9000 --contentFolder ./sdlc
+
+# Read-only mode: disables all write endpoints (POST/PUT/PATCH/DELETE)
+cnotes serve --read-only
+```
+
+**Built-in endpoints:**
+
+| Path | Description |
+|------|-------------|
+| `GET /api/inspect` | Collection overview |
+| `GET /api/models` | All model definitions |
+| `GET /api/documents` | List documents (filter with `?model=`) |
+| `GET/POST/PUT/PATCH/DELETE /api/documents/:pathId` | Document CRUD |
+| `GET /api/query?model=&where=&select=` | Query model instances (flat condition format) |
+| `POST /api/query` | Query with JSON DSL body (MongoDB-style) |
+| `GET /api/search?pattern=` | Full-text regex search across documents |
+| `GET /api/text-search?pattern=` | File-level text search (`expanded=true` for line detail) |
+| `GET /api/validate?pathId=` | Validate against schema |
+| `GET/POST /api/actions` | List or execute actions |
+| `GET /docs/:path.json\|.md\|.html` | Content-negotiated doc serving |
+| `GET /openapi.json` | Auto-generated OpenAPI 3.1 spec |
+
+When `--read-only` is passed, all mutating endpoints return `403 Forbidden`. This includes `POST /api/documents`, `PUT/PATCH/DELETE /api/documents/:pathId`, and `POST /api/actions`. Read endpoints are unaffected.
+
+You can also add your own endpoints by placing files in an `endpoints/` directory. See [CLI.md](./CLI.md#user-defined-endpoints) for details.
+
+### mcp
+
+Start a Model Context Protocol server for AI agent integration. Exposes tools, resources, and prompts for the collection.
 
 ```bash
-contentbase inspect           # show collection info
-contentbase validate          # validate all documents
-contentbase export            # export collection as JSON
-contentbase create Story      # scaffold a new document
-contentbase action publish    # run a named action
+cnotes mcp                                     # stdio transport (for Claude Desktop, etc.)
+cnotes mcp --transport http --port 3003        # HTTP transport
 ```
 
+### extract
+
+The `extract` command outputs document titles, leading content, and only the requested sections -- combined into a single document suitable for creating new content:
+
+```bash
+# Extract Acceptance Criteria from all stories
+cnotes extract "stories/**/*" --sections "Acceptance Criteria"
+
+# Combine epics into a single document with a title
+cnotes extract "epics/*" -s "Stories" --title "All Stories"
+
+# Multiple sections, include frontmatter, raw heading depths
+cnotes extract "epics/*" -s "Stories, Notes" --frontmatter --no-normalize-headings
+```
+
+Glob patterns are matched against document path IDs using [picomatch](https://github.com/micromatch/picomatch). Sections that don't exist in a document are silently skipped.
+
+By default, heading depths are normalized so each document's content nests properly in the combined output. When `--title` is provided, it becomes the h1 and document titles shift to h2. Use `--no-normalize-headings` to preserve original heading depths.
+
+### create
+
+The `create` command scaffolds new documents with smart defaults:
+
+```bash
+cnotes create Story --title "User can logout"
+cnotes create Epic --title "Payments" --meta.priority high
+```
+
+If a template exists at `templates/<model>.md` (or `.mdx`) in your content directory, it's used as the base. Meta values are merged with this priority: Zod defaults < model defaults < template frontmatter < CLI `--meta.*` flags.
+
+### Model Discovery
+
+The CLI uses a 3-tier system to find your models:
+
+**Tier 1 — `index.ts`** (recommended): If your content directory has an `index.ts` that exports a `Collection` with models registered, the CLI uses it directly. This is what `contentbase init` scaffolds.
+
+```ts
+// docs/index.ts
+import { Collection, defineModel, z } from "contentbase";
+
+const Post = defineModel("Post", {
+  meta: z.object({ draft: z.boolean().default(false) }),
+});
+
+export const collection = new Collection({ rootPath: import.meta.dir });
+collection.register(Post);
+```
+
+**Tier 2 — `models.ts`**: If no `index.ts` exists but a `models.ts` is found, the CLI imports it, detects model definitions from exports, and auto-registers them on a new Collection.
+
+**Tier 3 — Auto-discovery**: If neither file exists, the CLI scans top-level subdirectories for markdown files and generates bare models from folder names (`epics/` → `Epic`). These models have no schema validation — useful for quick inspection, but you'll want a `models.ts` or `index.ts` for real use.
+
 ---
 
 ## API Reference
@@ -557,18 +825,26 @@ contentbase action publish    # run a named action
 | --- | --- |
 | `Collection` | Loads and manages a directory of documents |
 | `Document` | A single Markdown/MDX file with AST operations |
-| `defineModel()` | Create a typed model definition |
+| `defineModel()` | Create a typed model definition (accepts optional `description`, auto-generated if omitted) |
+| `generateDescription()` | Generate a human-readable model description from its schema |
 | `section()` | Declare a section extraction |
 | `hasMany()` | Declare a one-to-many relationship |
 | `belongsTo()` | Declare a many-to-one relationship |
 | `parse()` | Parse a file path or markdown string into a queryable `ParsedDocument` |
 | `extractSections()` | Combine sections from multiple documents into one |
 | `CollectionQuery` | Fluent query builder for model instances |
+| `queryDSLSchema` | Zod schema for validating JSON query DSL input |
+| `parseWhereClause()` | Parse MongoDB-style where object into internal conditions |
+| `executeQueryDSL()` | Execute a JSON query DSL against a collection |
 | `AstQuery` | MDAST query wrapper (select, visit, find) |
 | `NodeShortcuts` | Convenience getters for common AST nodes |
 | `createModelInstance()` | Low-level factory for model instances |
 | `validateDocument()` | Standalone validation function |
+| `matchPattern()` | Express-style path pattern matching (`:param` syntax) |
+| `matchPatterns()` | Try multiple patterns against a path, first match wins |
+| `introspectMetaSchema()` | Extract field info (name, type, required, default) from a Zod schema |
 | `z` | Re-exported from Zod (no extra dependency needed) |
+| `toString` | Re-exported from `mdast-util-to-string` |
 
 ---
 

package/docs/README.md

@@ -0,0 +1,110 @@
+# Models
+
+Models define the structure of markdown documents in this collection. Each document is a markdown file with YAML frontmatter (metadata attributes) and a heading-based structure (sections). Models specify the expected frontmatter fields via a schema, named sections that map to `##` headings in the document body, relationships to other models, and computed properties derived at query time.
+
+## Examples
+
+An Example has metadata (tags).
+
+**Prefix:** `examples`
+
+### Attributes
+
+| Field | Type | Required | Default | Description |
+|-------|------|----------|---------|-------------|
+| tags | string[] | optional | `[]` | Arbitrary tags for categorizing the example |
+
+### Example
+
+```markdown
+---
+tags: []
+---
+# Example Title
+```
+
+---
+
+## Ideas
+
+An Idea has metadata (goal, tags, status).
+
+**Prefix:** `ideas`
+
+### Attributes
+
+| Field | Type | Required | Default | Description |
+|-------|------|----------|---------|-------------|
+| goal | string | optional | — | Slug of the goal this idea is aligned to |
+| tags | string[] | optional | `[]` | Arbitrary tags for categorizing the idea |
+| status | enum(`spark`, `exploring`, `parked`, `promoted`) | optional | `"spark"` | spark is a new raw idea, exploring means actively thinking about it, parked means on hold, promoted means it became a plan |
+
+### Example
+
+```markdown
+---
+tags: []
+status: spark
+---
+# Idea Title
+```
+
+---
+
+## Reports
+
+A Report has metadata (tags).
+
+**Prefix:** `reports`
+
+### Attributes
+
+| Field | Type | Required | Default | Description |
+|-------|------|----------|---------|-------------|
+| tags | string[] | optional | `[]` | Arbitrary tags for categorizing the report |
+
+### Example
+
+```markdown
+---
+tags: []
+---
+# Report Title
+```
+
+---
+
+## Tutorials
+
+A Tutorial has metadata (tags).
+
+**Prefix:** `tutorials`
+
+### Attributes
+
+| Field | Type | Required | Default | Description |
+|-------|------|----------|---------|-------------|
+| tags | string[] | optional | `[]` | Arbitrary tags for categorizing the tutorial |
+
+### Example
+
+```markdown
+---
+tags: []
+---
+# Tutorial Title
+```
+
+---
+
+## Bases
+
+A Base document.
+
+**Prefix:** ``
+
+### Example
+
+```markdown
+# Base Title
+```

package/docs/TABLE-OF-CONTENTS.md

@@ -0,0 +1,7 @@
+# Table of Contents
+
+## Ideas
+
+- [Changelogs](./ideas/changelogs.md)
+- [Content model collections](./ideas/content-model-collections.md)
+- [Semantic Search Integration](./ideas/semantic-search.md)

package/docs/models.ts

@@ -0,0 +1,38 @@
+import {
+  defineModel,
+  section,
+  hasMany,
+  belongsTo,
+  AstQuery,
+  z,
+} from "contentbase";
+
+export const Idea = defineModel("Idea", {
+	prefix: "ideas",
+	meta: z.object({
+		goal: z.string().optional().describe("Slug of the goal this idea is aligned to"),
+		tags: z.array(z.string()).default([]).describe("Arbitrary tags for categorizing the idea"),
+		status: z.enum(["spark", "exploring", "parked", "promoted"]).default("spark").describe("spark is a new raw idea, exploring means actively thinking about it, parked means on hold, promoted means it became a plan"),
+	}),
+});
+
+export const Tutorial = defineModel("Tutorial", {
+	prefix: "tutorials",
+	meta: z.object({
+		tags: z.array(z.string()).default([]).describe("Arbitrary tags for categorizing the tutorial"),
+	}),
+})
+
+export const Report = defineModel("Report", {
+	prefix: "reports",
+	meta: z.object({
+		tags: z.array(z.string()).default([]).describe("Arbitrary tags for categorizing the report"),
+	}),
+})
+
+export const Example = defineModel("Example", {
+	prefix: "examples",
+	meta: z.object({
+		tags: z.array(z.string()).default([]).describe("Arbitrary tags for categorizing the example"),
+	}),
+})

package/models.ts

@@ -0,0 +1,38 @@
+import {
+  defineModel,
+  section,
+  hasMany,
+  belongsTo,
+  AstQuery,
+  z,
+} from "contentbase";
+
+export const Idea = defineModel("Idea", {
+	prefix: "ideas",
+	meta: z.object({
+		goal: z.string().optional().describe("Slug of the goal this idea is aligned to"),
+		tags: z.array(z.string()).default([]).describe("Arbitrary tags for categorizing the idea"),
+		status: z.enum(["spark", "exploring", "parked", "promoted"]).default("spark").describe("spark is a new raw idea, exploring means actively thinking about it, parked means on hold, promoted means it became a plan"),
+	}),
+});
+
+export const Tutorial = defineModel("Tutorial", {
+	prefix: "tutorials",
+	meta: z.object({
+		tags: z.array(z.string()).default([]).describe("Arbitrary tags for categorizing the tutorial"),
+	}),
+})
+
+export const Report = defineModel("Report", {
+	prefix: "reports",
+	meta: z.object({
+		tags: z.array(z.string()).default([]).describe("Arbitrary tags for categorizing the report"),
+	}),
+})
+
+export const Example = defineModel("Example", {
+	prefix: "examples",
+	meta: z.object({
+		tags: z.array(z.string()).default([]).describe("Arbitrary tags for categorizing the example"),
+	}),
+})

package/package.json

@@ -1,27 +1,32 @@
 {
   "name": "contentbase",
-  "version": "0.0.2",
+  "version": "0.0.5",
   "type": "module",
   "main": "./src/index.ts",
   "types": "./src/index.ts",
   "bin": {
+    "cnotes": "./src/cli/index.ts",
     "contentbase": "./src/cli/index.ts"
   },
   "scripts": {
     "test": "vitest run",
     "test:watch": "vitest",
     "typecheck": "tsc --noEmit",
-    "examples": "bun run scripts/examples/run-all.ts"
+    "examples": "bun run scripts/examples/run-all.ts",
+    "compile": "bun build ./src/cli/index.ts --compile --outfile dist/cnotes --external @node-llama-cpp/*"
   },
   "dependencies": {
-    "citty": "^0.1.6",
+    "@soederpop/luca": "^0.0.2",
     "gray-matter": "^4.0.3",
     "js-yaml": "^4.1.0",
     "mdast-util-mdxjs-esm": "^2.0.1",
     "mdast-util-to-markdown": "^2.1.2",
     "mdast-util-to-string": "^4.0.0",
+    "picomatch": "^4.0.3",
+    "rehype-stringify": "^10.0.1",
     "remark-gfm": "^4.0.0",
     "remark-parse": "^11.0.0",
+    "remark-rehype": "^11.1.2",
     "remark-stringify": "^11.0.0",
     "unified": "^11.0.5",
     "unist-util-find-after": "^5.0.0",
@@ -35,8 +40,12 @@
   "devDependencies": {
     "@types/js-yaml": "^4.0.9",
     "@types/mdast": "^4.0.4",
+    "@types/picomatch": "^4.0.2",
     "bun-types": "^1.1.0",
     "typescript": "^5.4.0",
     "vitest": "^1.6.0"
+  },
+  "luca": {
+    "aliases": ["contentbase", "content base", "content", "the orm"]
   }
 }

package/src/api/endpoints/actions.ts

@@ -0,0 +1,34 @@
+import { z } from 'zod'
+
+export const path = '/api/actions'
+export const description = 'List or execute collection actions'
+export const tags = ['actions']
+
+export async function get(_params: any, ctx: any) {
+  const collection = ctx.container._contentbaseCollection
+  return { actions: collection.availableActions }
+}
+
+export const postSchema = z.object({
+  name: z.string(),
+  args: z.array(z.any()).optional(),
+})
+
+export async function post(params: any, ctx: any) {
+  if (ctx.container._contentbaseReadOnly) {
+    ctx.response.status(403)
+    return { error: 'Server is running in read-only mode' }
+  }
+
+  const collection = ctx.container._contentbaseCollection
+
+  if (!collection.availableActions.includes(params.name)) {
+    ctx.response.status(400)
+    return {
+      error: `Unknown action: ${params.name}. Available: ${collection.availableActions.join(', ') || '(none)'}`,
+    }
+  }
+
+  const result = await collection.runAction(params.name, ...(params.args || []))
+  return typeof result === 'string' ? { result } : result
+}