dirsql 0.3.6 → 0.3.8

package/README.md

@@ -17,6 +17,7 @@ Prebuilt binaries ship for linux-x64, linux-arm64, darwin-x64, darwin-arm64, and
 ## Usage
 
 ```typescript
+import { readFileSync } from "node:fs";
 import { DirSQL } from "dirsql";
 
 const db = new DirSQL({
@@ -25,7 +26,7 @@ const db = new DirSQL({
     {
       ddl: "CREATE TABLE users (name TEXT, age INTEGER)",
       glob: "data/*.json",
-      extract: (_filePath, content) => JSON.parse(content),
+      extract: (filePath) => JSON.parse(readFileSync(filePath, "utf8")),
     },
   ],
 });

package/dist/index.d.ts

@@ -4,8 +4,13 @@ export interface TableDef {
     ddl: string;
     /** Glob pattern (relative to the DirSQL root) for files backing this table. */
     glob: string;
-    /** Extract rows from a file's contents. Returns an array of row objects. */
-    extract: (filePath: string, content: string) => Record<string, unknown>[];
+    /**
+     * Produce the rows a matched file contributes. Receives the absolute
+     * filesystem path of the file. dirsql does not read file contents; if the
+     * callback needs the file body it reads the path itself (e.g.
+     * `fs.readFileSync(filePath, "utf8")`). Returns an array of row objects.
+     */
+    extract: (filePath: string) => Record<string, unknown>[];
     /** If true, reject rows with columns not declared in `ddl`. */
     strict?: boolean;
 }

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../ts/index.ts"],"names":[],"mappings":"AAcA,iEAAiE;AACjE,MAAM,WAAW,QAAQ;IACvB,6EAA6E;IAC7E,GAAG,EAAE,MAAM,CAAC;IACZ,+EAA+E;IAC/E,IAAI,EAAE,MAAM,CAAC;IACb,4EAA4E;IAC5E,OAAO,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,KAAK,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;IAC1E,+DAA+D;IAC/D,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED;;;;;;GAMG;AACH,MAAM,WAAW,aAAa;IAC5B,8BAA8B;IAC9B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,8EAA8E;IAC9E,MAAM,CAAC,EAAE,QAAQ,EAAE,CAAC;IACpB,oDAAoD;IACpD,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAClB;;;;;;OAMG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;;OAIG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,qDAAqD;AACrD,MAAM,WAAW,QAAQ;IACvB;;;;OAIG;IACH,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,MAAM,EAAE,QAAQ,GAAG,QAAQ,GAAG,QAAQ,GAAG,OAAO,CAAC;IACjD,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IACrC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IACxC,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC1B;AAGD,UAAU,YAAY;IACpB,KAAK,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC,CAAC;IACvD,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAC9B,UAAU,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;CACpD;AAED,UAAU,uBAAuB;IAC/B,SAAS,CACP,IAAI,EAAE,MAAM,GAAG,IAAI,EACnB,MAAM,EAAE,QAAQ,EAAE,GAAG,IAAI,EACzB,MAAM,EAAE,MAAM,EAAE,GAAG,IAAI,EACvB,MAAM,EAAE,MAAM,GAAG,IAAI,EACrB,OAAO,EAAE,OAAO,GAAG,IAAI,EACvB,WAAW,EAAE,MAAM,GAAG,IAAI,GACzB,OAAO,CAAC,YAAY,CAAC,CAAC;CAC1B;AAID,UAAU,UAAU;IAClB,MAAM,EAAE,uBAAuB,CAAC;CACjC;AAaD;;;;;;;GAOG;AACH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,UAAU,GAAG,IAAI,GAAG,IAAI,CAEjE;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,qBAAa,MAAM;IACjB;;;;;;OAMG;IACH,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;IAG9B,OAAO,CAAC,MAAM,CAAgB;IAE9B,wDAAwD;gBAC5C,UAAU,EAAE,MAAM;IAC9B,yCAAyC;gBAC7B,OAAO,EAAE,aAAa;IAkBlC;;;;;;OAMG;IACG,KAAK,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;IAK5D;;;;;;;OAOG;IACG,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;IAKnC;;;;;;;OAOG;IACG,UAAU,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;IAKxD;;;;;;;;;;OAUG;IACI,KAAK,IAAI,cAAc,CAAC,QAAQ,EAAE,IAAI,EAAE,OAAO,CAAC;CAaxD"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../ts/index.ts"],"names":[],"mappings":"AAcA,iEAAiE;AACjE,MAAM,WAAW,QAAQ;IACvB,6EAA6E;IAC7E,GAAG,EAAE,MAAM,CAAC;IACZ,+EAA+E;IAC/E,IAAI,EAAE,MAAM,CAAC;IACb;;;;;OAKG;IACH,OAAO,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;IACzD,+DAA+D;IAC/D,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED;;;;;;GAMG;AACH,MAAM,WAAW,aAAa;IAC5B,8BAA8B;IAC9B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,8EAA8E;IAC9E,MAAM,CAAC,EAAE,QAAQ,EAAE,CAAC;IACpB,oDAAoD;IACpD,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAClB;;;;;;OAMG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;;OAIG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,qDAAqD;AACrD,MAAM,WAAW,QAAQ;IACvB;;;;OAIG;IACH,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,MAAM,EAAE,QAAQ,GAAG,QAAQ,GAAG,QAAQ,GAAG,OAAO,CAAC;IACjD,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IACrC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IACxC,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC1B;AAGD,UAAU,YAAY;IACpB,KAAK,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC,CAAC;IACvD,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAC9B,UAAU,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;CACpD;AAED,UAAU,uBAAuB;IAC/B,SAAS,CACP,IAAI,EAAE,MAAM,GAAG,IAAI,EACnB,MAAM,EAAE,QAAQ,EAAE,GAAG,IAAI,EACzB,MAAM,EAAE,MAAM,EAAE,GAAG,IAAI,EACvB,MAAM,EAAE,MAAM,GAAG,IAAI,EACrB,OAAO,EAAE,OAAO,GAAG,IAAI,EACvB,WAAW,EAAE,MAAM,GAAG,IAAI,GACzB,OAAO,CAAC,YAAY,CAAC,CAAC;CAC1B;AAID,UAAU,UAAU;IAClB,MAAM,EAAE,uBAAuB,CAAC;CACjC;AAaD;;;;;;;GAOG;AACH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,UAAU,GAAG,IAAI,GAAG,IAAI,CAEjE;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,qBAAa,MAAM;IACjB;;;;;;OAMG;IACH,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;IAG9B,OAAO,CAAC,MAAM,CAAgB;IAE9B,wDAAwD;gBAC5C,UAAU,EAAE,MAAM;IAC9B,yCAAyC;gBAC7B,OAAO,EAAE,aAAa;IAkBlC;;;;;;OAMG;IACG,KAAK,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;IAK5D;;;;;;;OAOG;IACG,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;IAKnC;;;;;;;OAOG;IACG,UAAU,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;IAKxD;;;;;;;;;;OAUG;IACI,KAAK,IAAI,cAAc,CAAC,QAAQ,EAAE,IAAI,EAAE,OAAO,CAAC;CAaxD"}

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../ts/index.ts"],"names":[],"mappings":"AAAA,yBAAyB;AACzB,EAAE;AACF,wEAAwE;AACxE,kEAAkE;AAClE,qEAAqE;AACrE,qCAAqC;AACrC,EAAE;AACF,qEAAqE;AACrE,uEAAuE;AACvE,uEAAuE;AACvE,mEAAmE;AAEnE,OAAO,EAAE,cAAc,IAAI,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AAwF9E,yEAAyE;AACzE,sEAAsE;AACtE,IAAI,IAAI,GAAsB,IAAI,CAAC;AAEnC,SAAS,OAAO;IACd,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;QAClB,IAAI,GAAG,qBAAqB,EAAgB,CAAC;IAC/C,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,mBAAmB,CAAC,IAAuB;IACzD,IAAI,GAAG,IAAI,CAAC;AACd,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,MAAM,OAAO,MAAM;IACjB;;;;;;OAMG;IACM,KAAK,CAAgB;IAE9B,gEAAgE;IACxD,MAAM,CAAgB;IAM9B,YAAY,GAA2B;QACrC,MAAM,OAAO,GACX,OAAO,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;QAClD,MAAM,IAAI,GAAG,OAAO,EAAE,CAAC,MAAM,CAAC;QAC9B,MAAM,WAAW,GAAG,IAAI,CAAC,SAAS,CAChC,OAAO,CAAC,IAAI,IAAI,IAAI,EACpB,OAAO,CAAC,MAAM,IAAI,IAAI,EACtB,OAAO,CAAC,MAAM,IAAI,IAAI,EACtB,OAAO,CAAC,MAAM,IAAI,IAAI,EACtB,OAAO,CAAC,OAAO,IAAI,IAAI,EACvB,OAAO,CAAC,WAAW,IAAI,IAAI,CAC5B,CAAC;QACF,IAAI,CAAC,KAAK,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE;YACtC,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;QACtB,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,KAAK,CAAC,GAAW;QACrB,MAAM,IAAI,CAAC,KAAK,CAAC;QACjB,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAChC,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,YAAY;QAChB,MAAM,IAAI,CAAC,KAAK,CAAC;QACjB,OAAO,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC;IACpC,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,UAAU,CAAC,SAAiB;QAChC,MAAM,IAAI,CAAC,KAAK,CAAC;QACjB,OAAO,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;IAC3C,CAAC;IAED;;;;;;;;;;OAUG;IACH,KAAK,CAAC,CAAC,KAAK;QACV,MAAM,IAAI,CAAC,KAAK,CAAC;QACjB,MAAM,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC;QACjC,OAAO,IAAI,EAAE,CAAC;YACZ,qEAAqE;YACrE,gEAAgE;YAChE,sEAAsE;YACtE,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;YACjD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBAC3B,MAAM,KAAK,CAAC;YACd,CAAC;QACH,CAAC;IACH,CAAC;CACF"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../ts/index.ts"],"names":[],"mappings":"AAAA,yBAAyB;AACzB,EAAE;AACF,wEAAwE;AACxE,kEAAkE;AAClE,qEAAqE;AACrE,qCAAqC;AACrC,EAAE;AACF,qEAAqE;AACrE,uEAAuE;AACvE,uEAAuE;AACvE,mEAAmE;AAEnE,OAAO,EAAE,cAAc,IAAI,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AA6F9E,yEAAyE;AACzE,sEAAsE;AACtE,IAAI,IAAI,GAAsB,IAAI,CAAC;AAEnC,SAAS,OAAO;IACd,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;QAClB,IAAI,GAAG,qBAAqB,EAAgB,CAAC;IAC/C,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,mBAAmB,CAAC,IAAuB;IACzD,IAAI,GAAG,IAAI,CAAC;AACd,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,MAAM,OAAO,MAAM;IACjB;;;;;;OAMG;IACM,KAAK,CAAgB;IAE9B,gEAAgE;IACxD,MAAM,CAAgB;IAM9B,YAAY,GAA2B;QACrC,MAAM,OAAO,GACX,OAAO,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;QAClD,MAAM,IAAI,GAAG,OAAO,EAAE,CAAC,MAAM,CAAC;QAC9B,MAAM,WAAW,GAAG,IAAI,CAAC,SAAS,CAChC,OAAO,CAAC,IAAI,IAAI,IAAI,EACpB,OAAO,CAAC,MAAM,IAAI,IAAI,EACtB,OAAO,CAAC,MAAM,IAAI,IAAI,EACtB,OAAO,CAAC,MAAM,IAAI,IAAI,EACtB,OAAO,CAAC,OAAO,IAAI,IAAI,EACvB,OAAO,CAAC,WAAW,IAAI,IAAI,CAC5B,CAAC;QACF,IAAI,CAAC,KAAK,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE;YACtC,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;QACtB,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,KAAK,CAAC,GAAW;QACrB,MAAM,IAAI,CAAC,KAAK,CAAC;QACjB,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAChC,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,YAAY;QAChB,MAAM,IAAI,CAAC,KAAK,CAAC;QACjB,OAAO,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC;IACpC,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,UAAU,CAAC,SAAiB;QAChC,MAAM,IAAI,CAAC,KAAK,CAAC;QACjB,OAAO,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;IAC3C,CAAC;IAED;;;;;;;;;;OAUG;IACH,KAAK,CAAC,CAAC,KAAK;QACV,MAAM,IAAI,CAAC,KAAK,CAAC;QACjB,MAAM,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC;QACjC,OAAO,IAAI,EAAE,CAAC;YACZ,qEAAqE;YACrE,gEAAgE;YAChE,sEAAsE;YACtE,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;YACjD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBAC3B,MAAM,KAAK,CAAC;YACd,CAAC;QACH,CAAC;IACH,CAAC;CACF"}

package/docs/api/index.md

@@ -168,15 +168,15 @@ import { Table } from 'dirsql';
 ::: code-group
 
 ```python [Python]
-Table(*, ddl: str, glob: str, extract: Callable[[str, str], list[dict]])
+Table(*, ddl: str, glob: str, extract: Callable[[str], list[dict]])
 ```
 
 ```rust [Rust]
-Table::new(ddl: &str, glob: &str, extract: fn(&str, &str) -> Vec<Value>)
+Table::new(ddl: &str, glob: &str, extract: fn(&str) -> Vec<Value>)
 ```
 
 ```typescript [TypeScript]
-new Table({ ddl: string, glob: string, extract: (path: string, content: string) => Record<string, unknown>[] })
+new Table({ ddl: string, glob: string, extract: (path: string) => Record<string, unknown>[] })
 ```
 
 :::
@@ -187,7 +187,7 @@ Defines a mapping from files to SQLite table rows.
 
 - `ddl` -- A `CREATE TABLE` statement. The table name is parsed from this DDL.
 - `glob` -- A glob pattern matched against file paths relative to the root directory.
-- `extract` -- A callable `(path, content) -> list[dict]`. Receives the relative file path and file content as strings. Returns a list of dicts/maps mapping column names to values. Return an empty list to skip a file.
+- `extract` -- A callable `(path) -> list[dict]`. Receives the absolute filesystem path of the matched file. `dirsql` does not read file contents; a callback that needs the file body reads `path` itself. Returns a list of dicts/maps mapping column names to values. Return an empty list to skip a file.
 
 **Attributes:**
 

package/docs/getting-started.md

@@ -63,12 +63,12 @@ async def main():
             Table(
                 ddl="CREATE TABLE posts (title TEXT, author TEXT)",
                 glob="posts/*.json",
-                extract=lambda path, content: [json.loads(content)],
+                extract=lambda path: [json.loads(open(path, encoding="utf-8").read())],
             ),
             Table(
                 ddl="CREATE TABLE authors (id TEXT, name TEXT)",
                 glob="authors/*.json",
-                extract=lambda path, content: [json.loads(content)],
+                extract=lambda path: [json.loads(open(path, encoding="utf-8").read())],
             ),
         ],
     )
@@ -120,12 +120,12 @@ let db = DirSQL::new(
         Table::new(
             "CREATE TABLE posts (title TEXT, author TEXT)",
             "posts/*.json",
-            |_path, content| vec![row_from_json(content)],
+            |path| vec![row_from_json(&std::fs::read_to_string(path).unwrap())],
         ),
         Table::new(
             "CREATE TABLE authors (id TEXT, name TEXT)",
             "authors/*.json",
-            |_path, content| vec![row_from_json(content)],
+            |path| vec![row_from_json(&std::fs::read_to_string(path).unwrap())],
         ),
     ],
 )?;
@@ -139,18 +139,19 @@ let results = db.query(
 ```
 
 ```typescript [TypeScript]
+import { readFileSync } from 'node:fs';
 import { DirSQL, type TableDef } from 'dirsql';
 
 const tables: TableDef[] = [
   {
     ddl: 'CREATE TABLE posts (title TEXT, author TEXT)',
     glob: 'posts/*.json',
-    extract: (_path, content) => [JSON.parse(content)],
+    extract: (path) => [JSON.parse(readFileSync(path, 'utf8'))],
   },
   {
     ddl: 'CREATE TABLE authors (id TEXT, name TEXT)',
     glob: 'authors/*.json',
-    extract: (_path, content) => [JSON.parse(content)],
+    extract: (path) => [JSON.parse(readFileSync(path, 'utf8'))],
   },
 ];
 
@@ -169,8 +170,8 @@ const results = await db.query(`
 ## What happens at startup
 
 1. `dirsql` walks the directory tree
-2. Files matching each table's glob pattern are read
-3. The `extract` function converts file content into rows
+2. Files matching each table's glob pattern are identified
+3. The `extract` function receives each matched file's absolute path and returns rows
 4. Rows are inserted into an in-memory SQLite database
 5. SQL queries run against that database
 

package/docs/guide/async.md

@@ -24,7 +24,7 @@ async def main():
             Table(
                 ddl="CREATE TABLE items (name TEXT, value INTEGER)",
                 glob="data/*.json",
-                extract=lambda path, content: [json.loads(content)],
+                extract=lambda path: [json.loads(open(path, encoding="utf-8").read())],
             ),
         ],
     )
@@ -47,7 +47,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
             Table::new(
                 "CREATE TABLE items (name TEXT, value INTEGER)",
                 "data/*.json",
-                |_path, content| vec![serde_json::from_str(content).unwrap()],
+                |path| vec![serde_json::from_str(&std::fs::read_to_string(path).unwrap()).unwrap()],
             ),
         ],
     )?;
@@ -59,6 +59,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
 ```
 
 ```typescript [TypeScript]
+import { readFileSync } from 'node:fs';
 import { DirSQL, Table } from 'dirsql';
 
 const db = new DirSQL({
@@ -67,7 +68,7 @@ const db = new DirSQL({
     new Table({
       ddl: 'CREATE TABLE items (name TEXT, value INTEGER)',
       glob: 'data/*.json',
-      extract: (_path, content) => [JSON.parse(content)],
+      extract: (path) => [JSON.parse(readFileSync(path, 'utf8'))],
     }),
   ],
 });

package/docs/guide/crdt.md

@@ -60,7 +60,7 @@ db = DirSQL(
             ddl="CREATE TABLE posts (id TEXT, title TEXT, body TEXT, updated INTEGER)",
             # Match the JSON view, not the raw CRDT binary.
             glob="posts/*/view.json",
-            extract=lambda path, content: [json.loads(content)],
+            extract=lambda path: [json.loads(open(path, encoding="utf-8").read())],
         ),
     ],
 )
@@ -76,20 +76,21 @@ let db = DirSQL::new(
         Table::new(
             "CREATE TABLE posts (id TEXT, title TEXT, body TEXT, updated INTEGER)",
             "posts/*/view.json",
-            |_path, content| vec![row_from_json(content)],
+            |path| vec![row_from_json(&std::fs::read_to_string(path).unwrap())],
         ),
     ],
 )?;
 ```
 
 ```typescript [TypeScript]
+import { readFileSync } from 'node:fs';
 import { DirSQL, type TableDef } from 'dirsql';
 
 const tables: TableDef[] = [
   {
     ddl: 'CREATE TABLE posts (id TEXT, title TEXT, body TEXT, updated INTEGER)',
     glob: 'posts/*/view.json',
-    extract: (_path, content) => [JSON.parse(content)],
+    extract: (path) => [JSON.parse(readFileSync(path, 'utf8'))],
   },
 ];
 

package/docs/guide/tables.md

@@ -18,7 +18,7 @@ from dirsql import Table
 table = Table(
     ddl="CREATE TABLE comments (id TEXT, body TEXT, author TEXT)",
     glob="comments/**/index.jsonl",
-    extract=lambda path, content: [
+    extract=lambda path: [
         {"id": "...", "body": "...", "author": "..."}
     ],
 )
@@ -31,7 +31,7 @@ use std::collections::HashMap;
 let table = Table::new(
     "CREATE TABLE comments (id TEXT, body TEXT, author TEXT)",
     "comments/**/index.jsonl",
-    |_path, _content| {
+    |_path| {
         let mut row: HashMap<String, Value> = HashMap::new();
         row.insert("id".into(), Value::Text("...".into()));
         row.insert("body".into(), Value::Text("...".into()));
@@ -47,7 +47,7 @@ import type { TableDef } from 'dirsql';
 const table: TableDef = {
   ddl: 'CREATE TABLE comments (id TEXT, body TEXT, author TEXT)',
   glob: 'comments/**/index.jsonl',
-  extract: (_path, content) => [
+  extract: (_path) => [
     { id: '...', body: '...', author: '...' },
   ],
 };
@@ -89,38 +89,38 @@ Glob syntax follows standard Unix globbing rules. `**` matches any number of dir
 
 ### `extract`
 
-A callable `(path: str, content: str) -> list[dict]` that converts a file into rows.
+A callable `(path: str) -> list[dict]` that converts a file into rows.
 
-- `path` is the file path relative to the root directory
-- `content` is the file content as a string
+- `path` is the **absolute filesystem path** of the matched file
 - Return a list of dicts, where each dict maps column names to values
 - Return an empty list to skip a file
 
+`dirsql` does not read file contents for you. If your extract needs the file
+body, read it inside the callback using `path`. Callbacks that derive columns
+only from the path (or that rely solely on the auto-injected filesystem-fact
+columns) never touch the file at all.
+
 ```python
 import json
 
 # Single-object JSON files: one row per file
-extract=lambda path, content: [json.loads(content)]
+def extract(path):
+    with open(path, encoding="utf-8") as f:
+        return [json.loads(f.read())]
 
 # JSONL files: one row per line
-extract=lambda path, content: [
-    json.loads(line) for line in content.splitlines()
-]
+def extract(path):
+    with open(path, encoding="utf-8") as f:
+        return [json.loads(line) for line in f]
 
-# Derive values from the file path
+# Derive a value from the file path alone -- no file read
 import os
-extract=lambda path, content: [
-    {
-        "id": os.path.basename(os.path.dirname(path)),
-        "body": json.loads(line)["body"],
-    }
-    for line in content.splitlines()
-    for _ in [json.loads(line)]
-]
+extract = lambda path: [{"id": os.path.basename(os.path.dirname(path))}]
 
 # Conditionally skip files
-def extract(path, content):
-    data = json.loads(content)
+def extract(path):
+    with open(path, encoding="utf-8") as f:
+        data = json.loads(f.read())
     if data.get("draft"):
         return []
     return [data]
@@ -142,12 +142,12 @@ db = DirSQL(
         Table(
             ddl="CREATE TABLE posts (title TEXT, author_id TEXT)",
             glob="posts/*.json",
-            extract=lambda path, content: [json.loads(content)],
+            extract=lambda path: [json.loads(open(path, encoding="utf-8").read())],
         ),
         Table(
             ddl="CREATE TABLE authors (id TEXT, name TEXT)",
             glob="authors/*.json",
-            extract=lambda path, content: [json.loads(content)],
+            extract=lambda path: [json.loads(open(path, encoding="utf-8").read())],
         ),
     ],
 )
@@ -184,12 +184,12 @@ let db = DirSQL::new(
         Table::new(
             "CREATE TABLE posts (title TEXT, author_id TEXT)",
             "posts/*.json",
-            |_path, content| vec![row_from_json(content)],
+            |path| vec![row_from_json(&std::fs::read_to_string(path).unwrap())],
         ),
         Table::new(
             "CREATE TABLE authors (id TEXT, name TEXT)",
             "authors/*.json",
-            |_path, content| vec![row_from_json(content)],
+            |path| vec![row_from_json(&std::fs::read_to_string(path).unwrap())],
         ),
     ],
 )?;
@@ -197,17 +197,18 @@ let db = DirSQL::new(
 
 ```typescript [TypeScript]
 import { DirSQL, type TableDef } from 'dirsql';
+import { readFileSync } from 'node:fs';
 
 const tables: TableDef[] = [
   {
     ddl: 'CREATE TABLE posts (title TEXT, author_id TEXT)',
     glob: 'posts/*.json',
-    extract: (_path, content) => [JSON.parse(content)],
+    extract: (path) => [JSON.parse(readFileSync(path, 'utf8'))],
   },
   {
     ddl: 'CREATE TABLE authors (id TEXT, name TEXT)',
     glob: 'authors/*.json',
-    extract: (_path, content) => [JSON.parse(content)],
+    extract: (path) => [JSON.parse(readFileSync(path, 'utf8'))],
   },
 ];
 

package/docs/guide/watching.md

@@ -26,7 +26,7 @@ db = DirSQL(
         Table(
             ddl="CREATE TABLE comments (id TEXT, body TEXT, author TEXT)",
             glob="comments/**/*.json",
-            extract=lambda path, content: [json.loads(content)],
+            extract=lambda path: [json.loads(open(path, encoding="utf-8").read())],
         ),
     ],
 )
@@ -67,7 +67,7 @@ let db = DirSQL::new(
         Table::new(
             "CREATE TABLE comments (id TEXT, body TEXT, author TEXT)",
             "comments/**/*.json",
-            |_path, content| vec![row_from_json(content)],
+            |path| vec![row_from_json(&std::fs::read_to_string(path).unwrap())],
         ),
     ],
 )?;
@@ -92,13 +92,14 @@ while let Some(event) = stream.next().await {
 ```
 
 ```typescript [TypeScript]
+import { readFileSync } from 'node:fs';
 import { DirSQL, type TableDef } from 'dirsql';
 
 const tables: TableDef[] = [
   {
     ddl: 'CREATE TABLE comments (id TEXT, body TEXT, author TEXT)',
     glob: 'comments/**/*.json',
-    extract: (_path, content) => [JSON.parse(content)],
+    extract: (path) => [JSON.parse(readFileSync(path, 'utf8'))],
   },
 ];
 

package/docs/index.md

@@ -35,7 +35,7 @@ db = DirSQL(
         Table(
             ddl="CREATE TABLE files (name TEXT, size INTEGER, type TEXT)",
             glob="data/*.json",
-            extract=lambda path, content: [json.loads(content)],
+            extract=lambda path: [json.loads(open(path, encoding="utf-8").read())],
         ),
     ],
 )
@@ -53,7 +53,7 @@ let db = DirSQL::new(
         Table::new(
             "CREATE TABLE files (name TEXT, size INTEGER, type TEXT)",
             "data/*.json",
-            |_path, content| vec![serde_json::from_str(content).unwrap()],
+            |path| vec![serde_json::from_str(&std::fs::read_to_string(path).unwrap()).unwrap()],
         ),
     ],
 )?;
@@ -62,6 +62,7 @@ let large = db.query("SELECT * FROM files WHERE size > 1000")?;
 ```
 
 ```typescript [TypeScript]
+import { readFileSync } from 'node:fs';
 import { DirSQL, Table } from 'dirsql';
 
 const db = new DirSQL({
@@ -70,7 +71,7 @@ const db = new DirSQL({
     new Table({
       ddl: 'CREATE TABLE files (name TEXT, size INTEGER, type TEXT)',
       glob: 'data/*.json',
-      extract: (_path, content) => [JSON.parse(content)],
+      extract: (path) => [JSON.parse(readFileSync(path, 'utf8'))],
     }),
   ],
 });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dirsql",
-  "version": "0.3.6",
+  "version": "0.3.8",
   "description": "Ephemeral SQL index over a local directory",
   "license": "MIT",
   "repository": "https://github.com/thekevinscott/dirsql",
@@ -191,15 +191,15 @@
     ]
   },
   "optionalDependencies": {
-    "@dirsql/lib-linux-x64-gnu": "0.3.6",
-    "@dirsql/lib-linux-arm64-gnu": "0.3.6",
-    "@dirsql/lib-darwin-x64": "0.3.6",
-    "@dirsql/lib-darwin-arm64": "0.3.6",
-    "@dirsql/lib-win32-x64-msvc": "0.3.6",
-    "@dirsql/cli-linux-x64-gnu": "0.3.6",
-    "@dirsql/cli-linux-arm64-gnu": "0.3.6",
-    "@dirsql/cli-darwin-x64": "0.3.6",
-    "@dirsql/cli-darwin-arm64": "0.3.6",
-    "@dirsql/cli-win32-x64-msvc": "0.3.6"
+    "@dirsql/lib-linux-x64-gnu": "0.3.8",
+    "@dirsql/lib-linux-arm64-gnu": "0.3.8",
+    "@dirsql/lib-darwin-x64": "0.3.8",
+    "@dirsql/lib-darwin-arm64": "0.3.8",
+    "@dirsql/lib-win32-x64-msvc": "0.3.8",
+    "@dirsql/cli-linux-x64-gnu": "0.3.8",
+    "@dirsql/cli-linux-arm64-gnu": "0.3.8",
+    "@dirsql/cli-darwin-x64": "0.3.8",
+    "@dirsql/cli-darwin-arm64": "0.3.8",
+    "@dirsql/cli-win32-x64-msvc": "0.3.8"
   }
 }