datanotes 2.0.0 → 3.0.0

package/dist/docs/guides/read-and-write/import-notes.md

@@ -0,0 +1,896 @@
+# Import existing notes
+
+> Turn a folder of notes you already have into a table: infer its columns, create it and adopt the notes. Then choose which new notes become rows on their own.
+
+Your vault may already have notes with front matter, like a film diary, a reading list or meeting notes. The examples start from a `Films/` folder of classic films written by hand, each with its year, runtime and `Director`. You turn it into a table without retyping anything: you infer the columns, create the table and adopt the notes. Then you decide which new notes become rows on their own.
+
+## Infer a table from notes
+
+`infer_table` reads the notes’ front matter and proposes a table, without writing anything. The proposal has:
+
+* **A type for each key**: an enum when a few short values repeat across enough notes
+* **Shared keys**: which front matter keys every note has
+* **Valid column names**: a key whose name isn’t a valid column name, like `Director`, gets a valid name and a `from`
+
+**`infer_table`**
+
+```json
+{ "folder": "Films", "name": "film" }
+```
+
+Answer (153 lines)
+
+```json
+{
+  "files": 5,
+  "paths": [
+    "Films/Charade.md",
+    "Films/His Girl Friday.md",
+    "Films/Metropolis.md",
+    "Films/Nosferatu.md",
+    "Films/The Gold Rush.md"
+  ],
+  "skipped": [],
+  "columns": [
+    {
+      "name": "director",
+      "from": "Director",
+      "type": "string",
+      "description": null,
+      "stats": {
+        "present": 5,
+        "filled": 5,
+        "distinct": 5,
+        "types": { "string": 5 },
+        "examples": ["Stanley Donen", "Howard Hawks", "Fritz Lang"]
+      },
+      "required": true,
+      "unique": true,
+      "dense": true
+    },
+    {
+      "name": "minutes",
+      "type": "integer",
+      "description": null,
+      "stats": {
+        "present": 5,
+        "filled": 5,
+        "distinct": 5,
+        "types": { "integer": 5 },
+        "examples": [113, 92, 153]
+      },
+      "required": true,
+      "unique": true,
+      "dense": true
+    },
+    {
+      "name": "status",
+      "type": "enum",
+      "description": null,
+      "stats": {
+        "present": 5,
+        "filled": 5,
+        "distinct": 2,
+        "types": { "string": 5 },
+        "examples": ["watched", "watched", "to watch"]
+      },
+      "values": ["watched", "to watch"],
+      "required": true,
+      "dense": true
+    },
+    {
+      "name": "year",
+      "type": "integer",
+      "description": null,
+      "stats": {
+        "present": 5,
+        "filled": 5,
+        "distinct": 5,
+        "types": { "integer": 5 },
+        "examples": [1963, 1940, 1927]
+      },
+      "required": true,
+      "unique": true,
+      "dense": true
+    },
+    {
+      "name": "rating",
+      "type": "integer",
+      "description": null,
+      "stats": {
+        "present": 4,
+        "filled": 4,
+        "distinct": 2,
+        "types": { "integer": 4 },
+        "examples": [4, 5, 5]
+      }
+    },
+    {
+      "name": "watched_on",
+      "type": "date",
+      "description": null,
+      "stats": {
+        "present": 4,
+        "filled": 4,
+        "distinct": 4,
+        "types": { "date": 4 },
+        "examples": ["2026-09-12", "2026-07-20", "2026-03-14"]
+      }
+    }
+  ],
+  "proposal": {
+    "name": "film",
+    "description": null,
+    "columns": [
+      {
+        "name": "director",
+        "type": "string",
+        "description": null,
+        "required": true,
+        "unique": true,
+        "dense": true
+      },
+      {
+        "name": "minutes",
+        "type": "integer",
+        "description": null,
+        "required": true,
+        "unique": true,
+        "dense": true
+      },
+      {
+        "name": "status",
+        "type": "enum",
+        "description": null,
+        "values": ["watched", "to watch"],
+        "required": true,
+        "dense": true
+      },
+      {
+        "name": "year",
+        "type": "integer",
+        "description": null,
+        "required": true,
+        "unique": true,
+        "dense": true
+      },
+      { "name": "rating", "type": "integer", "description": null },
+      { "name": "watched_on", "type": "date", "description": null }
+    ]
+  },
+  "adopt": {
+    "paths": [
+      "Films/Charade.md",
+      "Films/His Girl Friday.md",
+      "Films/Metropolis.md",
+      "Films/Nosferatu.md",
+      "Films/The Gold Rush.md"
+    ],
+    "rename": { "Director": "director" },
+    "relocate": false
+  },
+  "hints": [
+    "no column matches the file names: set filename (e.g. \"{title}\") and adopt with relocate: false to keep the current names",
+    "write the description of the table and of each column (null in the proposal) before create_table: it refuses them empty"
+  ]
+}
+```
+
+Read the proposal and its `notes` before you use it. The notes can’t say what the table and its columns mean. So `description` is `null` on the table and on every column, for you to write. Here five columns also need a fix:
+
+* **`year`**, **`minutes`** and **`director`**: came out `unique` because no two of these films share them, which won’t stay true
+* **`status`**: an enum with its values in the order the notes gave them, `watched` first, where you want an ordered enum from `to watch` to `watched`
+* **`rating`**: an integer with no range, where you want 1 to 5
+* **`title`**: missing, because the notes keep the title in the note’s name, not in the front matter
+
+## Create the table from the proposal
+
+`create_table` refuses the proposal as it is, because its descriptions are `null`. The refusal names every one you still have to write:
+
+**`create_table`**
+
+```json
+{
+  "name": "film",
+  "description": null,
+  "columns": [
+    {
+      "name": "director",
+      "type": "string",
+      "description": null,
+      "required": true,
+      "unique": true,
+      "dense": true
+    },
+    {
+      "name": "minutes",
+      "type": "integer",
+      "description": null,
+      "required": true,
+      "unique": true,
+      "dense": true
+    },
+    {
+      "name": "status",
+      "type": "enum",
+      "description": null,
+      "values": ["watched", "to watch"],
+      "required": true,
+      "dense": true
+    },
+    {
+      "name": "year",
+      "type": "integer",
+      "description": null,
+      "required": true,
+      "unique": true,
+      "dense": true
+    },
+    { "name": "rating", "type": "integer", "description": null },
+    { "name": "watched_on", "type": "date", "description": null }
+  ]
+}
+```
+
+Refused:
+
+```text
+pass "time_column" or "filename" (e.g. "{name}") so rows get file names
+Hint: Check the arguments of create_table: name (string, required), description (string), columns (list of objects, required), catalog (string), schema (string), time_column (string), filename (string), partition (string), exclude (list of strings), archive (string), aliases (list of strings), history (boolean), autolink_target (boolean), row_template (string), fields (object), tags (object), dry_run (boolean).
+Docs: https://datanotes.dev/reference/operations/schema.md#create_table (offline: read_docs("reference/operations/schema#create_table"))
+```
+
+This call adds a description to the table and to each column, and corrects what the inference guessed wrong:
+
+* **`year`**, **`minutes`** and **`director`**: no longer unique
+* **`status`**: an ordered enum (`to watch` < `watched`)
+* **`rating`**: from 1 to 5
+* **`title`**: a new column that holds the note’s file name (`identity: "filename"`)
+
+**`create_table`**
+
+```json
+{
+  "name": "film",
+  "description": "Classic films I watched or want to watch",
+  "columns": [
+    {
+      "name": "title",
+      "type": "string",
+      "identity": "filename",
+      "description": "The film's title, which is also its note's name"
+    },
+    {
+      "name": "director",
+      "type": "string",
+      "description": "Who directed the film",
+      "required": true,
+      "unique": false,
+      "dense": true
+    },
+    {
+      "name": "minutes",
+      "type": "integer",
+      "description": "Running time in minutes",
+      "required": true,
+      "unique": false,
+      "dense": true
+    },
+    {
+      "name": "status",
+      "type": "enum",
+      "description": "Where the film is on my list",
+      "values": ["to watch", "watched"],
+      "required": true,
+      "dense": true,
+      "ordered": true
+    },
+    {
+      "name": "year",
+      "type": "integer",
+      "description": "Year the film first came out",
+      "required": true,
+      "unique": false,
+      "dense": true
+    },
+    {
+      "name": "rating",
+      "type": "integer",
+      "min": 1,
+      "max": 5,
+      "description": "My rating, from 1 (poor) to 5 (a masterpiece)"
+    },
+    { "name": "watched_on", "type": "date", "description": "The day I last watched it" }
+  ],
+  "filename": "{title}"
+}
+```
+
+Answer:
+
+```json
+{
+  "ok": true,
+  "op": "create_table",
+  "planned_texts": ["datanotes/metastore.yaml"],
+  "commits": [{ "table": "main.public.film", "version": 0, "operation": "CREATE TABLE" }],
+  "migration_id": "20261012T071957-create_table-create-table-main-public-film"
+}
+```
+
+## Preview the adoption with a dry run
+
+`adopt_rows` makes existing notes rows of a table and keeps their text and links. It sets `table` and an `id` on each note, and applies these arguments:
+
+* **`rename`**: renames front matter keys to column names
+* **`set`**: sets the same values on every note
+* **`drop`**: removes the front matter keys the table doesn’t have
+* **`relocate`**: moves the notes to the table’s folder and rewrites the links to them
+
+With `dry_run`, it shows what it would do and writes nothing.
+
+**`adopt_rows`**
+
+```json
+{
+  "table": "film",
+  "paths": [
+    "Films/Charade.md",
+    "Films/His Girl Friday.md",
+    "Films/Metropolis.md",
+    "Films/Nosferatu.md",
+    "Films/The Gold Rush.md"
+  ],
+  "rename": { "Director": "director" },
+  "relocate": true,
+  "dry_run": true
+}
+```
+
+Answer (70 lines)
+
+```json
+{
+  "ok": true,
+  "op": "adopt_rows",
+  "dry_run": true,
+  "planned_edits": [
+    {
+      "path": "Films/Charade.md",
+      "set": {
+        "table": "main.public.film",
+        "id": "01a12efb-81a4-717d-a664-f60882bb8996",
+        "director": "Stanley Donen",
+        "title": "Charade"
+      },
+      "remove": ["Director"]
+    },
+    {
+      "path": "Films/His Girl Friday.md",
+      "set": {
+        "table": "main.public.film",
+        "id": "01a12efb-81a4-7e0e-8864-4d93d474be34",
+        "director": "Howard Hawks",
+        "title": "His Girl Friday"
+      },
+      "remove": ["Director"]
+    },
+    {
+      "path": "Films/Metropolis.md",
+      "set": {
+        "table": "main.public.film",
+        "id": "01a12efb-81a4-747e-8aef-048494381ac2",
+        "director": "Fritz Lang",
+        "title": "Metropolis"
+      },
+      "remove": ["Director"]
+    },
+    {
+      "path": "Films/Nosferatu.md",
+      "set": {
+        "table": "main.public.film",
+        "id": "01a12efb-81a4-71bf-9b8f-a28f778461db",
+        "director": "F. W. Murnau",
+        "title": "Nosferatu"
+      },
+      "remove": ["Director"]
+    },
+    {
+      "path": "Films/The Gold Rush.md",
+      "set": {
+        "table": "main.public.film",
+        "id": "01a12efb-81a4-7628-817c-ce1281d27329",
+        "director": "Charlie Chaplin",
+        "title": "The Gold Rush"
+      },
+      "remove": ["Director"]
+    }
+  ],
+  "planned_moves": [
+    { "from": "Films/Charade.md", "to": "datanotes/main/public/film/Charade.md" },
+    {
+      "from": "Films/His Girl Friday.md",
+      "to": "datanotes/main/public/film/His Girl Friday.md"
+    },
+    { "from": "Films/Metropolis.md", "to": "datanotes/main/public/film/Metropolis.md" },
+    { "from": "Films/Nosferatu.md", "to": "datanotes/main/public/film/Nosferatu.md" },
+    {
+      "from": "Films/The Gold Rush.md",
+      "to": "datanotes/main/public/film/The Gold Rush.md"
+    }
+  ]
+}
+```
+
+## Adopt the notes into the table
+
+This is the same call without `dry_run`. The engine checks every note first: a note that doesn’t fit the table stops the whole adoption, and the answer lists why.
+
+With thousands of notes, adopt them in calls of about 1,000 `paths` each. Each call is one journal entry, so a stop or a crash interrupts less, and `undo_migration` reverts one chunk at a time.
+
+**`adopt_rows`**
+
+```json
+{
+  "table": "film",
+  "paths": [
+    "Films/Charade.md",
+    "Films/His Girl Friday.md",
+    "Films/Metropolis.md",
+    "Films/Nosferatu.md",
+    "Films/The Gold Rush.md"
+  ],
+  "rename": { "Director": "director" },
+  "relocate": true
+}
+```
+
+Answer (73 lines)
+
+```json
+{
+  "ok": true,
+  "op": "adopt_rows",
+  "notes_changed": 5,
+  "files_moved": 5,
+  "planned_edits": [
+    {
+      "path": "Films/Charade.md",
+      "set": {
+        "table": "main.public.film",
+        "id": "01a12efb-81ad-7350-b6fc-3607a2c36814",
+        "director": "Stanley Donen",
+        "title": "Charade"
+      },
+      "remove": ["Director"]
+    },
+    {
+      "path": "Films/His Girl Friday.md",
+      "set": {
+        "table": "main.public.film",
+        "id": "01a12efb-81ad-76cb-a7d9-1d8bcd578f6d",
+        "director": "Howard Hawks",
+        "title": "His Girl Friday"
+      },
+      "remove": ["Director"]
+    },
+    {
+      "path": "Films/Metropolis.md",
+      "set": {
+        "table": "main.public.film",
+        "id": "01a12efb-81ad-7b8f-aa3d-16ff4a7685be",
+        "director": "Fritz Lang",
+        "title": "Metropolis"
+      },
+      "remove": ["Director"]
+    },
+    {
+      "path": "Films/Nosferatu.md",
+      "set": {
+        "table": "main.public.film",
+        "id": "01a12efb-81ad-7a1c-b7d7-c6ce93fd565e",
+        "director": "F. W. Murnau",
+        "title": "Nosferatu"
+      },
+      "remove": ["Director"]
+    },
+    {
+      "path": "Films/The Gold Rush.md",
+      "set": {
+        "table": "main.public.film",
+        "id": "01a12efb-81ad-7816-bbb2-0c7ac62d182a",
+        "director": "Charlie Chaplin",
+        "title": "The Gold Rush"
+      },
+      "remove": ["Director"]
+    }
+  ],
+  "planned_moves": [
+    { "from": "Films/Charade.md", "to": "datanotes/main/public/film/Charade.md" },
+    {
+      "from": "Films/His Girl Friday.md",
+      "to": "datanotes/main/public/film/His Girl Friday.md"
+    },
+    { "from": "Films/Metropolis.md", "to": "datanotes/main/public/film/Metropolis.md" },
+    { "from": "Films/Nosferatu.md", "to": "datanotes/main/public/film/Nosferatu.md" },
+    {
+      "from": "Films/The Gold Rush.md",
+      "to": "datanotes/main/public/film/The Gold Rush.md"
+    }
+  ],
+  "commits": [{ "table": "main.public.film", "version": 1, "operation": "ADOPT ROWS" }],
+  "migration_id": "20261012T071957-adopt_rows-adopt-5-notes-into-film"
+}
+```
+
+The notes keep their text and names, so `[[His Girl Friday]]` in Charade still resolves:
+
+**`read_note`**
+
+```json
+{ "path": "datanotes/main/public/film/Charade.md" }
+```
+
+Answer:
+
+```markdown
+---
+table: main.public.film
+id: 01a12efb-81ad-7350-b6fc-3607a2c36814
+year: 1963
+minutes: 113
+status: watched
+rating: 4
+watched_on: 2026-09-12
+director: Stanley Donen
+title: Charade
+---
+Cary Grant again, twenty years after [[His Girl Friday]], and just as quick.
+```
+
+**`select_rows`**
+
+```json
+{ "table": "film", "columns": ["title", "status", "rating"], "order_by": "title" }
+```
+
+Answer:
+
+```json
+{
+  "table": "main.public.film",
+  "total": 5,
+  "rows": [
+    {
+      "path": "datanotes/main/public/film/Charade.md",
+      "title": "Charade",
+      "status": "watched",
+      "rating": 4
+    },
+    {
+      "path": "datanotes/main/public/film/His Girl Friday.md",
+      "title": "His Girl Friday",
+      "status": "watched",
+      "rating": 5
+    },
+    {
+      "path": "datanotes/main/public/film/Metropolis.md",
+      "title": "Metropolis",
+      "status": "to watch",
+      "rating": null
+    },
+    {
+      "path": "datanotes/main/public/film/Nosferatu.md",
+      "title": "Nosferatu",
+      "status": "watched",
+      "rating": 5
+    },
+    {
+      "path": "datanotes/main/public/film/The Gold Rush.md",
+      "title": "The Gold Rush",
+      "status": "watched",
+      "rating": 5
+    }
+  ]
+}
+```
+
+## Insert new rows through the engine
+
+`title` is the note’s file name (`identity: "filename"`). When you give it at insert, it names the new note, and the engine then keeps it equal to the note’s actual name.
+
+**`insert_row`**
+
+```json
+{
+  "table": "film",
+  "values": {
+    "title": "Battleship Potemkin",
+    "year": 1925,
+    "minutes": 75,
+    "director": "Sergei Eisenstein",
+    "status": "to watch"
+  }
+}
+```
+
+Answer:
+
+```json
+{
+  "ok": true,
+  "op": "insert_row",
+  "files_created": 1,
+  "planned_creates": ["datanotes/main/public/film/Battleship Potemkin.md"],
+  "commits": [{ "table": "main.public.film", "version": 2, "operation": "WRITE" }],
+  "migration_id": "20261012T071957-insert_row-insert-into-film"
+}
+```
+
+## Set a value for notes made by hand with `external_default`
+
+`default` fills a column on rows the engine writes. `external_default` fills it on notes that become rows without the engine writing them, like the note in the next example. Here `added_by` tells the two apart.
+
+**`add_column`**
+
+```json
+{
+  "table": "film",
+  "column": {
+    "name": "added_by",
+    "type": "enum",
+    "values": ["agent", "by hand"],
+    "default": "agent",
+    "external_default": "by hand",
+    "description": "Who added the film: an agent through the engine, or a person in an editor"
+  }
+}
+```
+
+Answer:
+
+```json
+{
+  "ok": true,
+  "op": "add_column",
+  "planned_texts": ["datanotes/metastore.yaml"],
+  "commits": [{ "table": "main.public.film", "version": 4, "operation": "ADD COLUMNS" }],
+  "migration_id": "20261012T071958-add_column-add-column-added_by-to-film"
+}
+```
+
+**`insert_row`**
+
+```json
+{
+  "table": "film",
+  "values": {
+    "title": "Night of the Living Dead",
+    "year": 1968,
+    "minutes": 96,
+    "director": "George A. Romero",
+    "status": "to watch"
+  }
+}
+```
+
+Answer:
+
+```json
+{
+  "ok": true,
+  "op": "insert_row",
+  "files_created": 1,
+  "planned_creates": ["datanotes/main/public/film/Night of the Living Dead.md"],
+  "commits": [{ "table": "main.public.film", "version": 5, "operation": "WRITE" }],
+  "migration_id": "20261012T071958-insert_row-insert-into-film"
+}
+```
+
+`datanotes/main/public/film/Night of the Living Dead.md` now reads:
+
+```markdown
+---
+table: main.public.film
+id: 01a12efb-87b5-7e4d-95f6-04c05995e9df
+director: George A. Romero
+minutes: 96
+status: to watch
+year: 1968
+title: Night of the Living Dead
+added_by: agent
+---
+```
+
+## A new note in a table’s folder becomes a row
+
+The engine adopts a note you create by hand inside a table’s folder, for example with “new note” in a note editor such as Obsidian, or from a daily note template. The note gets:
+
+* `table` and an `id`
+* the values its name carries under the table’s `filename` pattern
+* the column defaults and the `external_default` values
+
+Here the note’s name gives `title`, and `added_by` is `by hand`. A note that misses a required column or has a bad value stays as it is, and the engine logs a warning.
+
+You edit `datanotes/main/public/film/Sherlock Jr..md` by hand, outside the engine. The note now reads:
+
+```markdown
+---
+year: 1924
+minutes: 45
+status: to watch
+director: Buster Keaton
+---
+```
+
+**`read_note`**
+
+```json
+{ "path": "datanotes/main/public/film/Sherlock Jr..md" }
+```
+
+Answer:
+
+```markdown
+---
+table: main.public.film
+id: 01a12efb-8a37-7b33-bb22-169f2afb9934
+year: 1924
+minutes: 45
+status: to watch
+director: Buster Keaton
+added_by: by hand
+title: Sherlock Jr.
+---
+```
+
+## Keep a folder’s notes out of the table with `exclude`
+
+A note is a row when its front matter key `table` names the table, and a new note in the table’s folder becomes one. `exclude` lists folders whose notes never become rows. Here it lists the editor’s templates folder. The template already says `table:`, so the notes you make from it are rows, but the template itself must not be one.
+
+**`alter_table`**
+
+```json
+{ "table": "film", "exclude": ["datanotes/main/public/film/Templates"] }
+```
+
+Answer:
+
+```json
+{
+  "ok": true,
+  "op": "alter_table",
+  "planned_texts": ["datanotes/metastore.yaml"],
+  "commits": [{ "table": "main.public.film", "version": 7, "operation": "ALTER TABLE" }],
+  "migration_id": "20261012T072001-alter_table-alter-table-main-public-film"
+}
+```
+
+You edit `datanotes/main/public/film/Templates/New film.md` by hand, outside the engine. The note now reads:
+
+```markdown
+---
+table: main.public.film
+year:
+status: to watch
+---
+## Why I want to watch it
+```
+
+The engine leaves the template as it is, with no `id`, and it isn’t a row:
+
+**`read_note`**
+
+```json
+{ "path": "datanotes/main/public/film/Templates/New film.md" }
+```
+
+Answer:
+
+```markdown
+---
+table: main.public.film
+year:
+status: to watch
+---
+## Why I want to watch it
+```
+
+**`select_rows`**
+
+```json
+{
+  "table": "film",
+  "columns": ["title"],
+  "where": { "path": { "contains": "Templates" } }
+}
+```
+
+Answer:
+
+```json
+{ "table": "main.public.film", "total": 0, "rows": [] }
+```
+
+## See the table’s definition in `metastore.yaml`
+
+This is the table as you inferred, corrected and extended it on this page. Its definition holds:
+
+* `title` with `identity: filename`
+* the columns from the notes’ front matter
+* `added_by` with its `external_default`
+* the `exclude` folder under `properties`
+
+`datanotes/metastore.yaml` now reads:
+
+```yaml
+# Metastore (Datanotes): catalogs, schemas, tables, columns, views.
+id: 01a12efb-8191-7a5c-9aee-8ffe9c6f215a
+config:
+  history: true
+
+
+catalogs:
+  main:
+    schemas:
+      public:
+        tables:
+          film:
+            description: Classic films I watched or want to watch
+            options:
+              filename: "{title}"
+              archive: _archive
+              exclude: datanotes/main/public/film/Templates
+            columns:
+              - name: id
+                type: string
+                unique: true
+                identity: uuid
+                required: true
+                dense: true
+                description: "Unique id of the row (UUIDv7): set when the row is created, never changes"
+              - name: title
+                type: string
+                description: The film's title, which is also its note's name
+                identity: filename
+              - name: director
+                type: string
+                required: true
+                description: Who directed the film
+                dense: true
+              - name: minutes
+                type: integer
+                required: true
+                description: Running time in minutes
+                dense: true
+              - name: status
+                type: enum
+                required: true
+                values:
+                  - to watch
+                  - watched
+                description: Where the film is on my list
+                ordered: true
+                dense: true
+              - name: year
+                type: integer
+                required: true
+                description: Year the film first came out
+                dense: true
+              - name: rating
+                type: integer
+                min: 1
+                max: 5
+                description: My rating, from 1 (poor) to 5 (a masterpiece)
+              - name: watched_on
+                type: date
+                description: The day I last watched it
+              - name: added_by
+                type: enum
+                default: agent
+                values:
+                  - agent
+                  - by hand
+                external_default: by hand
+                description: "Who added the film: an agent through the engine, or a person in an editor"
+```