dirsql 0.2.3 → 0.2.8

package/docs/guide/tables.md

@@ -0,0 +1,268 @@
+---
+canonical: https://thekevinscott.github.io/dirsql/guide/tables
+---
+
+# Defining Tables
+
+> Online: <https://thekevinscott.github.io/dirsql/guide/tables>
+
+Each table in `dirsql` maps a set of files to rows in an in-memory SQLite table. A table definition has three parts: DDL, a glob pattern, and an extract function.
+
+## Table constructor
+
+::: code-group
+
+```python [Python]
+from dirsql import Table
+
+table = Table(
+    ddl="CREATE TABLE comments (id TEXT, body TEXT, author TEXT)",
+    glob="comments/**/index.jsonl",
+    extract=lambda path, content: [
+        {"id": "...", "body": "...", "author": "..."}
+    ],
+)
+```
+
+```rust [Rust]
+use dirsql::{Table, Value};
+use std::collections::HashMap;
+
+let table = Table::new(
+    "CREATE TABLE comments (id TEXT, body TEXT, author TEXT)",
+    "comments/**/index.jsonl",
+    |_path, _content| {
+        let mut row: HashMap<String, Value> = HashMap::new();
+        row.insert("id".into(), Value::Text("...".into()));
+        row.insert("body".into(), Value::Text("...".into()));
+        row.insert("author".into(), Value::Text("...".into()));
+        vec![row]
+    },
+);
+```
+
+```typescript [TypeScript]
+import type { TableDef } from 'dirsql';
+
+const table: TableDef = {
+  ddl: 'CREATE TABLE comments (id TEXT, body TEXT, author TEXT)',
+  glob: 'comments/**/index.jsonl',
+  extract: (_path, content) => [
+    { id: '...', body: '...', author: '...' },
+  ],
+};
+```
+
+:::
+
+All three arguments are keyword-only (in Python). In Rust they are positional to `Table::new`. In TypeScript a table is a plain `TableDef` object literal — the TS SDK exports the `TableDef` type (not a class).
+
+### `ddl`
+
+A SQLite `CREATE TABLE` statement. This defines the schema of the table. `dirsql` executes this DDL directly against the in-memory database, so any valid SQLite column types and constraints work.
+
+```python
+# Simple text columns
+ddl="CREATE TABLE notes (title TEXT, body TEXT)"
+
+# Typed columns
+ddl="CREATE TABLE metrics (name TEXT, value REAL, count INTEGER)"
+
+# With constraints
+ddl="CREATE TABLE items (id TEXT PRIMARY KEY, name TEXT NOT NULL)"
+```
+
+The table name is parsed from the DDL. It must be a valid SQLite identifier.
+
+### `glob`
+
+A glob pattern that determines which files feed into this table. Matched relative to the root directory passed to `DirSQL`.
+
+```python
+glob="*.json"                  # JSON files in root only
+glob="**/*.json"               # JSON files at any depth
+glob="comments/**/index.jsonl" # JSONL files in comment subdirectories
+glob="data/*.csv"              # CSV files in data/
+```
+
+Glob syntax follows standard Unix globbing rules. `**` matches any number of directory levels.
+
+### `extract`
+
+A callable `(path: str, content: str) -> list[dict]` that converts a file into rows.
+
+- `path` is the file path relative to the root directory
+- `content` is the file content as a string
+- Return a list of dicts, where each dict maps column names to values
+- Return an empty list to skip a file
+
+```python
+import json
+
+# Single-object JSON files: one row per file
+extract=lambda path, content: [json.loads(content)]
+
+# JSONL files: one row per line
+extract=lambda path, content: [
+    json.loads(line) for line in content.splitlines()
+]
+
+# Derive values from the file path
+import os
+extract=lambda path, content: [
+    {
+        "id": os.path.basename(os.path.dirname(path)),
+        "body": json.loads(line)["body"],
+    }
+    for line in content.splitlines()
+    for _ in [json.loads(line)]
+]
+
+# Conditionally skip files
+def extract(path, content):
+    data = json.loads(content)
+    if data.get("draft"):
+        return []
+    return [data]
+```
+
+## Multiple tables
+
+Pass multiple `Table` definitions to index different file types into separate tables:
+
+::: code-group
+
+```python [Python]
+from dirsql import DirSQL, Table
+import json
+
+db = DirSQL(
+    "./workspace",
+    tables=[
+        Table(
+            ddl="CREATE TABLE posts (title TEXT, author_id TEXT)",
+            glob="posts/*.json",
+            extract=lambda path, content: [json.loads(content)],
+        ),
+        Table(
+            ddl="CREATE TABLE authors (id TEXT, name TEXT)",
+            glob="authors/*.json",
+            extract=lambda path, content: [json.loads(content)],
+        ),
+    ],
+)
+```
+
+```rust [Rust]
+use dirsql::{DirSQL, Table, Value};
+use std::collections::HashMap;
+
+// See `row_from_json` in getting-started.md for a reusable helper.
+fn row_from_json(raw: &str) -> HashMap<String, Value> {
+    let v: serde_json::Value = serde_json::from_str(raw).unwrap();
+    let serde_json::Value::Object(obj) = v else { return HashMap::new() };
+    obj.into_iter()
+        .map(|(k, val)| {
+            let v = match val {
+                serde_json::Value::String(s) => Value::Text(s),
+                serde_json::Value::Number(n) => n
+                    .as_i64()
+                    .map(Value::Integer)
+                    .unwrap_or_else(|| Value::Real(n.as_f64().unwrap_or(0.0))),
+                serde_json::Value::Bool(b) => Value::Integer(b as i64),
+                serde_json::Value::Null => Value::Null,
+                other => Value::Text(other.to_string()),
+            };
+            (k, v)
+        })
+        .collect()
+}
+
+let db = DirSQL::new(
+    "./workspace",
+    vec![
+        Table::new(
+            "CREATE TABLE posts (title TEXT, author_id TEXT)",
+            "posts/*.json",
+            |_path, content| vec![row_from_json(content)],
+        ),
+        Table::new(
+            "CREATE TABLE authors (id TEXT, name TEXT)",
+            "authors/*.json",
+            |_path, content| vec![row_from_json(content)],
+        ),
+    ],
+)?;
+```
+
+```typescript [TypeScript]
+import { DirSQL, type TableDef } from 'dirsql';
+
+const tables: TableDef[] = [
+  {
+    ddl: 'CREATE TABLE posts (title TEXT, author_id TEXT)',
+    glob: 'posts/*.json',
+    extract: (_path, content) => [JSON.parse(content)],
+  },
+  {
+    ddl: 'CREATE TABLE authors (id TEXT, name TEXT)',
+    glob: 'authors/*.json',
+    extract: (_path, content) => [JSON.parse(content)],
+  },
+];
+
+const db = new DirSQL({ root: './workspace', tables });
+```
+
+:::
+
+Each table has its own glob and extract function. A file can only match one table (the first matching glob wins).
+
+## Ignore patterns
+
+Use the `ignore` parameter to exclude paths from all tables:
+
+::: code-group
+
+```python [Python]
+db = DirSQL(
+    "./workspace",
+    ignore=["**/node_modules/**", "**/.git/**"],
+    tables=[...],
+)
+```
+
+```rust [Rust]
+let db = DirSQL::with_ignore(
+    "./workspace",
+    vec![/* tables */],
+    vec!["**/node_modules/**", "**/.git/**"],
+)?;
+```
+
+```typescript [TypeScript]
+const db = new DirSQL({
+  root: './workspace',
+  tables: [/* tables */],
+  ignore: ['**/node_modules/**', '**/.git/**'],
+});
+```
+
+:::
+
+Ignore patterns are applied before glob matching. Any file matching an ignore pattern is skipped regardless of table globs.
+
+## Supported value types
+
+The extract function can return these Python types, which map to SQLite types:
+
+| Python type | SQLite type |
+|-------------|-------------|
+| `str`       | TEXT        |
+| `int`       | INTEGER     |
+| `float`     | REAL        |
+| `bool`      | INTEGER (0/1) |
+| `bytes`     | BLOB        |
+| `None`      | NULL        |
+
+Any other type is converted to its string representation via `str()`.

package/docs/guide/watching.md

@@ -0,0 +1,264 @@
+---
+canonical: https://thekevinscott.github.io/dirsql/guide/watching
+---
+
+# File Watching
+
+> Online: <https://thekevinscott.github.io/dirsql/guide/watching>
+
+`dirsql` can monitor the filesystem for changes and emit events when rows are inserted, updated, or deleted. This is useful for building reactive applications that respond to file changes in real time.
+
+## Starting a watch stream
+
+::: code-group
+
+```python [Python]
+from dirsql import DirSQL, Table
+import json
+
+db = DirSQL(
+    "./my-project",
+    tables=[
+        Table(
+            ddl="CREATE TABLE comments (id TEXT, body TEXT, author TEXT)",
+            glob="comments/**/*.json",
+            extract=lambda path, content: [json.loads(content)],
+        ),
+    ],
+)
+
+async for event in db.watch():
+    print(f"{event.action} on {event.table}: {event.row}")
+```
+
+```rust [Rust]
+use dirsql::{DirSQL, RowEvent, Table, Value};
+use futures::StreamExt;
+use std::collections::HashMap;
+
+// See `row_from_json` in getting-started.md for a reusable helper.
+fn row_from_json(raw: &str) -> HashMap<String, Value> {
+    let v: serde_json::Value = serde_json::from_str(raw).unwrap();
+    let serde_json::Value::Object(obj) = v else { return HashMap::new() };
+    obj.into_iter()
+        .map(|(k, val)| {
+            let v = match val {
+                serde_json::Value::String(s) => Value::Text(s),
+                serde_json::Value::Number(n) => n
+                    .as_i64()
+                    .map(Value::Integer)
+                    .unwrap_or_else(|| Value::Real(n.as_f64().unwrap_or(0.0))),
+                serde_json::Value::Bool(b) => Value::Integer(b as i64),
+                serde_json::Value::Null => Value::Null,
+                other => Value::Text(other.to_string()),
+            };
+            (k, v)
+        })
+        .collect()
+}
+
+let db = DirSQL::new(
+    "./my-project",
+    vec![
+        Table::new(
+            "CREATE TABLE comments (id TEXT, body TEXT, author TEXT)",
+            "comments/**/*.json",
+            |_path, content| vec![row_from_json(content)],
+        ),
+    ],
+)?;
+
+let mut stream = db.watch()?;
+while let Some(event) = stream.next().await {
+    match event {
+        RowEvent::Insert { table, row, file_path } => {
+            println!("insert on {table} ({file_path}): {row:?}")
+        }
+        RowEvent::Update { table, old_row, new_row, file_path } => {
+            println!("update on {table} ({file_path}): {old_row:?} -> {new_row:?}")
+        }
+        RowEvent::Delete { table, row, file_path } => {
+            println!("delete on {table} ({file_path}): {row:?}")
+        }
+        RowEvent::Error { file_path, error } => {
+            println!("error on {file_path:?}: {error}")
+        }
+    }
+}
+```
+
+```typescript [TypeScript]
+import { DirSQL, type TableDef } from 'dirsql';
+
+const tables: TableDef[] = [
+  {
+    ddl: 'CREATE TABLE comments (id TEXT, body TEXT, author TEXT)',
+    glob: 'comments/**/*.json',
+    extract: (_path, content) => [JSON.parse(content)],
+  },
+];
+
+const db = new DirSQL({ root: './my-project', tables });
+
+for await (const event of db.watch()) {
+  console.log(`${event.action} on ${event.table}:`, event.row);
+}
+```
+
+:::
+
+See [Async API](./async.md) for full details on the async `DirSQL` API (Python).
+
+## Event types
+
+Each event is a `RowEvent` object with these attributes:
+
+### `insert`
+
+A new row was added. This happens when a new file is created or an existing file gains additional rows.
+
+::: code-group
+
+```python [Python]
+event.action   # "insert"
+event.table    # "comments"
+event.row      # {"id": "abc", "body": "new comment", "author": "alice"}
+event.old_row  # None
+event.file_path # "comments/abc/index.json"
+```
+
+```rust [Rust]
+// RowEvent is an enum; match on the variant to destructure its fields.
+RowEvent::Insert {
+    table,     // "comments"
+    row,       // {"id": "abc", "body": "new comment", "author": "alice"}
+    file_path, // "comments/abc/index.json"
+} => { /* ... */ }
+```
+
+```typescript [TypeScript]
+event.action   // 'insert'
+event.table    // 'comments'
+event.row      // { id: 'abc', body: 'new comment', author: 'alice' }
+event.oldRow   // undefined
+event.filePath // 'comments/abc/index.json'
+```
+
+:::
+
+### `update`
+
+An existing row was modified. `dirsql` diffs the old and new rows extracted from the file to detect changes.
+
+::: code-group
+
+```python [Python]
+event.action   # "update"
+event.table    # "comments"
+event.row      # {"id": "abc", "body": "edited comment", "author": "alice"}
+event.old_row  # {"id": "abc", "body": "original comment", "author": "alice"}
+event.file_path # "comments/abc/index.json"
+```
+
+```rust [Rust]
+RowEvent::Update {
+    table,     // "comments"
+    old_row,   // {"id": "abc", "body": "original comment", "author": "alice"}
+    new_row,   // {"id": "abc", "body": "edited comment", "author": "alice"}
+    file_path, // "comments/abc/index.json"
+} => { /* ... */ }
+```
+
+```typescript [TypeScript]
+event.action   // 'update'
+event.table    // 'comments'
+event.row      // { id: 'abc', body: 'edited comment', author: 'alice' }
+event.oldRow   // { id: 'abc', body: 'original comment', author: 'alice' }
+event.filePath // 'comments/abc/index.json'
+```
+
+:::
+
+### `delete`
+
+A row was removed. This happens when a file is deleted or a file is modified to contain fewer rows.
+
+::: code-group
+
+```python [Python]
+event.action   # "delete"
+event.table    # "comments"
+event.row      # {"id": "abc", "body": "deleted comment", "author": "alice"}
+event.old_row  # None
+event.file_path # "comments/abc/index.json"
+```
+
+```rust [Rust]
+RowEvent::Delete {
+    table,     // "comments"
+    row,       // {"id": "abc", "body": "deleted comment", "author": "alice"}
+    file_path, // "comments/abc/index.json"
+} => { /* ... */ }
+```
+
+```typescript [TypeScript]
+event.action   // 'delete'
+event.table    // 'comments'
+event.row      // { id: 'abc', body: 'deleted comment', author: 'alice' }
+event.oldRow   // undefined
+event.filePath // 'comments/abc/index.json'
+```
+
+:::
+
+### `error`
+
+An error occurred while processing a file change. The file was modified but the extract function failed, or the file could not be read.
+
+::: code-group
+
+```python [Python]
+event.action    # "error"
+event.table     # "comments" (or None if the error isn't tied to a table)
+event.error     # "Extract error: ..."
+event.file_path # "comments/abc/index.json"
+event.row       # None
+```
+
+```rust [Rust]
+// `table` is `Option<String>`: `Some("comments")` when the failing
+// file matched a table's glob; `None` for errors that aren't tied
+// to a specific table (e.g. a watch-channel failure).
+RowEvent::Error {
+    table,     // Some("comments")
+    file_path, // PathBuf, e.g. "comments/abc/index.json"
+    error,     // "Extract error: ..."
+} => { /* ... */ }
+```
+
+```typescript [TypeScript]
+event.action   // 'error'
+event.table    // 'comments' (or null if the error isn't tied to a table)
+event.error    // 'Extract error: ...'
+event.filePath // 'comments/abc/index.json'
+event.row      // undefined
+```
+
+:::
+
+## How diffing works
+
+When a file changes, `dirsql`:
+
+1. Re-reads the file and calls the extract function to get new rows
+2. Compares new rows against the previously extracted rows for that file
+3. Emits insert, update, and delete events based on the diff
+4. Updates the in-memory database to reflect the new state
+
+Row identity is determined by position (row index within the file). If a file previously produced 3 rows and now produces 2, the first two rows are compared for updates and the third is emitted as a delete.
+
+## Filesystem events
+
+Under the hood, `dirsql` uses the `notify` crate (inotify on Linux, FSEvents on macOS, ReadDirectoryChangesW on Windows) to receive filesystem events. Events are coalesced and filtered through the table matcher before being processed.
+
+Files that do not match any table glob or that match an ignore pattern are silently skipped.

package/docs/index.md

@@ -0,0 +1,81 @@
+---
+canonical: https://thekevinscott.github.io/dirsql/
+layout: home
+hero:
+  name: dirsql
+  tagline: Ephemeral SQL index over a local directory. `dirsql` watches a filesystem, ingests structured files into an in-memory SQLite database, and exposes a SQL query interface. The filesystem is always the source of truth.
+  actions:
+    - theme: brand
+      text: Get Started
+      link: /getting-started
+    - theme: alt
+      text: GitHub
+      link: https://github.com/thekevinscott/dirsql
+---
+
+Structured data stored as flat files (JSON, CSV, markdown) is easy to read, write, diff, and version-control.
+
+But querying across many files is slow.
+
+"Show me all records matching X across 50 files" requires opening and parsing every file.
+
+## Solution
+
+`dirsql` bridges this gap. The filesystem remains the source of truth, but you get SQL queries and real-time change events for free. Define tables with glob patterns and extract functions, and `dirsql` handles the rest.
+
+::: code-group
+
+```python [Python]
+from dirsql import DirSQL, Table
+import json
+
+db = DirSQL(
+    "./my-project",
+    tables=[
+        Table(
+            ddl="CREATE TABLE files (name TEXT, size INTEGER, type TEXT)",
+            glob="data/*.json",
+            extract=lambda path, content: [json.loads(content)],
+        ),
+    ],
+)
+
+# SQL queries over your filesystem
+large = db.query("SELECT * FROM files WHERE size > 1000")
+```
+
+```rust [Rust]
+use dirsql::{DirSQL, Table};
+
+let db = DirSQL::new(
+    "./my-project",
+    vec![
+        Table::new(
+            "CREATE TABLE files (name TEXT, size INTEGER, type TEXT)",
+            "data/*.json",
+            |_path, content| vec![serde_json::from_str(content).unwrap()],
+        ),
+    ],
+)?;
+
+let large = db.query("SELECT * FROM files WHERE size > 1000")?;
+```
+
+```typescript [TypeScript]
+import { DirSQL, Table } from 'dirsql';
+
+const db = new DirSQL({
+  root: './my-project',
+  tables: [
+    new Table({
+      ddl: 'CREATE TABLE files (name TEXT, size INTEGER, type TEXT)',
+      glob: 'data/*.json',
+      extract: (_path, content) => [JSON.parse(content)],
+    }),
+  ],
+});
+
+const large = await db.query('SELECT * FROM files WHERE size > 1000');
+```
+
+:::

package/docs/migrations.md

@@ -0,0 +1,8 @@
+---
+title: Migrations
+canonical: https://thekevinscott.github.io/dirsql/migrations
+---
+
+> Online: <https://thekevinscott.github.io/dirsql/migrations>
+
+<!--@include: ../MIGRATIONS.md-->

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dirsql",
-  "version": "0.2.3",
+  "version": "0.2.8",
   "description": "Ephemeral SQL index over a local directory",
   "license": "MIT",
   "repository": "https://github.com/thekevinscott/dirsql",
@@ -18,7 +18,7 @@
   },
   "files": [
     "dist/",
-    "dirsql.node",
+    "docs/",
     "README.md",
     "LICENSE"
   ],
@@ -33,8 +33,9 @@
     "artifacts": "napi artifacts",
     "build": "wireit",
     "build:debug": "wireit",
-    "build:platforms": "tsx tools/buildPlatforms.ts",
-    "prepublishOnly": "napi prepublish -t npm",
+    "stage:platform": "wireit",
+    "prepack": "tsx tools/stageDocs.ts pre",
+    "postpack": "tsx tools/stageDocs.ts post",
     "test": "wireit",
     "test:unit": "wireit",
     "test:integration": "wireit",
@@ -62,10 +63,29 @@
         "dist/**"
       ]
     },
+    "stage:platform": {
+      "command": "tsx tools/stagePlatform.ts",
+      "dependencies": [
+        "napi:build"
+      ],
+      "files": [
+        "tools/stagePlatform.ts",
+        "ts/platforms.ts",
+        "src/**/*.rs",
+        "build.rs",
+        "Cargo.toml",
+        "../rust/src/**/*.rs",
+        "../rust/Cargo.toml"
+      ],
+      "output": [
+        "build/**"
+      ]
+    },
     "build": {
       "dependencies": [
         "napi:build",
-        "tsc"
+        "tsc",
+        "stage:platform"
       ]
     },
     "build:debug": {
@@ -154,5 +174,16 @@
       "esbuild"
     ]
   },
-  "optionalDependencies": {}
-}
+  "optionalDependencies": {
+    "@dirsql/lib-linux-x64-gnu": "0.2.8",
+    "@dirsql/lib-linux-arm64-gnu": "0.2.8",
+    "@dirsql/lib-darwin-x64": "0.2.8",
+    "@dirsql/lib-darwin-arm64": "0.2.8",
+    "@dirsql/lib-win32-x64-msvc": "0.2.8",
+    "@dirsql/cli-linux-x64-gnu": "0.2.8",
+    "@dirsql/cli-linux-arm64-gnu": "0.2.8",
+    "@dirsql/cli-darwin-x64": "0.2.8",
+    "@dirsql/cli-darwin-arm64": "0.2.8",
+    "@dirsql/cli-win32-x64-msvc": "0.2.8"
+  }
+}