@rejot-dev/tree-sitter-thalo 0.0.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 - present ReJot Nederland B.V., and individual contributors.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,377 @@
+# @rejot-dev/tree-sitter-thalo
+
+A Tree-Sitter grammar for parsing **Thalo** entries used in the Knowledge Center.
+
+## Overview
+
+Thalo is a syntax for recording structured knowledge entries including lore, opinions, references,
+and journal entries. It also supports a meta-layer for defining entity schemas.
+
+## Markdown Integration
+
+Thalo is designed to coexist with markdown. You can embed thalo code blocks inside markdown files
+using fenced code blocks with the `thalo` language identifier:
+
+````markdown
+# My Document
+
+Some markdown content here.
+
+```thalo
+2026-01-05T18:00Z create lore "An insight" #example
+  type: "insight"
+  subject: ^self
+
+  # Summary
+  This thalo entry lives inside a markdown file.
+```
+
+More markdown content.
+````
+
+When using the `@rejot-dev/thalo-prettier` plugin, Prettier automatically formats Thalo code blocks
+embedded in markdown files. This enables documentation files to include properly formatted Thalo
+examples.
+
+## Instance Entries
+
+Create or update instances of entities (lore, opinion, reference, journal):
+
+```
+{timestamp} {directive} {entity} "Title" [^link-id] [#tags...]
+  {key}: {value}
+  ...
+
+  {content}
+```
+
+### Example
+
+```thalo
+2026-01-05T18:11Z create lore "Custom event streaming system" ^event-streaming #architecture #distributed
+  type: "fact"
+  subject: ^acme-corp
+  date: 2018 ~ 2022
+
+  # Summary
+  The company built a custom event streaming system on top of Postgres before Kafka became widely
+  adopted.
+```
+
+### Header Line
+
+| Element   | Pattern                                   | Required | Example            |
+| --------- | ----------------------------------------- | -------- | ------------------ |
+| Timestamp | `YYYY-MM-DDTHH:MM`                        | Yes      | `2026-01-05T18:11` |
+| Directive | `create` or `update`                      | Yes      | `create`           |
+| Entity    | `lore`, `opinion`, `reference`, `journal` | Yes      | `lore`             |
+| Title     | Quoted string                             | Yes      | `"My title"`       |
+| Link      | `^` + identifier                          | No       | `^my-linked-entry` |
+| Tags      | `#` + identifier                          | No       | `#architecture`    |
+
+### Metadata
+
+Indented key-value pairs (2-space indent):
+
+```
+  key: "value"
+  ref-type: "article"
+  related: ^other-entry
+  source: "Technical documentation"
+  updated: 2026-01-05T18:11
+```
+
+- **Keys**: lowercase, may contain hyphens/underscores
+- **Values**: quoted strings, links (`^id`), timestamps, date ranges, or queries (no plain text)
+
+### Content
+
+Indented content after a blank line separator:
+
+```
+  # Section Header
+  Regular paragraph text continues here
+  across multiple lines.
+
+  # Another Section
+  More content with markdown-style headers.
+```
+
+- Content lines must be indented (2 spaces)
+- Markdown headers (`#`, `##`, etc.) are recognized
+- Blank lines within content are preserved
+
+## Schema Entries
+
+Define or alter entity schemas using `define-entity` and `alter-entity` directives:
+
+```
+{timestamp} define-entity {entity-name} "Description" [#tags...]
+  # Metadata
+  {field-name}?: {type} [= {default}] [; "description"]
+  ...
+  # Sections
+  {SectionName}? [; "description"]
+  ...
+```
+
+### Example
+
+```thalo
+2026-01-05T18:12Z define-entity reference "Collected resources"
+  # Metadata
+  url?: string ; "the url to the resource"
+  ref-type: "article" | "video" | "tweet"
+  author?: string | link
+  status?: "unread" | "read" = "unread"
+  related?: link[]
+
+  # Sections
+  Summary ; "Brief summary of the content"
+  KeyTakeaways?
+```
+
+### alter-entity
+
+Modify existing entity schemas by adding or removing fields/sections:
+
+```thalo
+2026-01-10T14:00Z alter-entity reference "Add published field, remove legacy"
+  # Metadata
+  published: datetime ; "publication date"
+  # Remove Metadata
+  legacy-field ; "deprecated in favor of new-field"
+
+  # Sections
+  New Section? ; "added section"
+
+  # Remove Sections
+  Old Section ; "no longer needed"
+```
+
+### Type System (Schema Definitions)
+
+| Type         | Example              | Description              |
+| ------------ | -------------------- | ------------------------ |
+| `string`     | `name: string`       | Free-form text           |
+| `date`       | `published: date`    | Single date              |
+| `date-range` | `period: date-range` | Date range (2022 ~ 2024) |
+| `link`       | `related: link`      | Reference to entry       |
+| Literal      | `status: "read"`     | Exact string value       |
+| Union        | `type: "a" \| "b"`   | One of multiple types    |
+| Array        | `tags: string[]`     | Array of type            |
+| Default      | `status?: "a" = "a"` | Default value            |
+
+## Typed Metadata Values
+
+The grammar parses metadata values into typed AST nodes, enabling downstream validation without
+regex-based parsing. All values must be explicitly typed (no plain/unquoted values).
+
+### Links
+
+Single link references:
+
+```text
+subject: ^self
+supersedes: ^previous-opinion
+```
+
+**AST node**: `link`
+
+### Quoted Values
+
+Values in double quotes (required for all string values including literal types):
+
+```text
+type: "fact"
+confidence: "high"
+description: "A longer text value"
+```
+
+**AST node**: `quoted_value`
+
+### Datetime Values
+
+Date or datetime values (date with optional time):
+
+```text
+published: 2026-01-07
+updated: 2026-01-07T12:00
+created: 2026-01-05T18:11
+```
+
+**AST node**: `datetime_value`
+
+### Date Ranges
+
+Date ranges with the `~` separator:
+
+```text
+date: 2022 ~ 2024
+period: 2022-05 ~ 2024-12-31
+```
+
+**AST node**: `date_range`
+
+### Query Expressions
+
+Source queries for synthesis entries:
+
+```text
+sources: lore where subject = ^self and #career
+sources: lore where type = "fact"
+```
+
+**AST structure**: `query` → `query_conditions` → `query_condition`
+
+Query conditions support:
+
+- **Field conditions**: `field = "quoted value"` or `field = ^link`
+- **Tag conditions**: `#tag`
+- **Link conditions**: `^link-id`
+
+### Arrays (Unified)
+
+Comma-separated lists of any value type (links, quoted values, timestamps, date ranges, or queries):
+
+```text
+related: ^ref1, ^ref2, ^ref3
+authors: "Jane Doe", ^john-ref, "Alice Smith"
+periods: 2020 ~ 2022, 2023 ~ 2024
+sources: lore where #career, journal where #reflection
+```
+
+**AST structure**: `value_array` → `(link | quoted_value | datetime_value | date_range | query)*`
+
+### Field Syntax
+
+- **Required by default**: Fields without `?` are required
+- **Optional marker**: `?` after field name makes it optional
+- **Description**: `; "text"` adds documentation
+
+### Section Names
+
+- Must start with uppercase letter (PascalCase)
+- Examples: `Summary`, `KeyTakeaways`, `Reasoning`
+
+## AST Structure
+
+### Instance Entry
+
+```
+source_file
+└── entry
+    └── instance_entry
+        ├── instance_header
+        │   ├── timestamp
+        │   ├── instance_directive
+        │   ├── entity
+        │   ├── title
+        │   ├── link?
+        │   └── tag*
+        ├── metadata*
+        │   ├── key
+        │   └── value
+        └── content?
+            ├── markdown_header*
+            └── content_line*
+```
+
+### Schema Entry
+
+```
+source_file
+└── entry
+    └── schema_entry
+        ├── schema_header
+        │   ├── timestamp
+        │   ├── schema_directive
+        │   ├── identifier
+        │   ├── title
+        │   ├── link?
+        │   └── tag*
+        ├── metadata_block?
+        │   └── field_definition*
+        │       ├── field_name
+        │       ├── optional_marker?
+        │       ├── type_expression
+        │       ├── default_value?
+        │       └── description?
+        ├── sections_block?
+        │   └── section_definition*
+        │       ├── section_name
+        │       ├── optional_marker?
+        │       └── description?
+        ├── remove_metadata_block?
+        │   └── field_removal*
+        └── remove_sections_block?
+            └── section_removal*
+```
+
+## Usage
+
+```bash
+# Generate parser
+pnpm exec tree-sitter generate
+
+# Run tests
+pnpm exec tree-sitter test
+
+# Parse a file
+pnpm exec tree-sitter parse path/to/file.thalo
+```
+
+## Limitations
+
+### General
+
+- Titles cannot contain unescaped quotes
+- Content text starting with `#` is always parsed as a markdown header
+- Only 2-space indentation is supported
+- Section names must be PascalCase
+- Field names must be lowercase (kebab-case or camelCase)
+
+### Typed Value Parsing
+
+- **All string values must be quoted**: There are no plain/unquoted values. Literal types like
+  `"fact"` require quotes.
+
+  ```text
+  # Correct:
+  type: "fact"
+  description: "Some text"
+  ```
+
+- **No inline comments in values**: Comments (`//`) after metadata values break parsing. Use a
+  separate comment line instead.
+
+  ```text
+  # Wrong - causes parse error:
+  type: "fact" // this breaks
+
+  # Correct:
+  // Note about type
+  type: "fact"
+  ```
+
+- **Single dates must be quoted**: The grammar only recognizes date ranges (`YYYY ~ YYYY`). Single
+  dates should be quoted strings.
+
+  ```text
+  # Correct:
+  published: "2024-05-11"
+  period: 2022 ~ 2024
+  ```
+
+- **Query `where` clause is required**: Queries must include a `where` clause.
+
+  ```text
+  # Correct:
+  sources: lore where #career
+
+  # Wrong - not a valid query:
+  sources: lore
+  ```
+
+- **Empty values cause parse errors**: Metadata must have a value; use optional fields and omit the
+  field entirely instead.

package/binding.gyp

@@ -0,0 +1,35 @@
+{
+  "targets": [
+    {
+      "target_name": "tree_sitter_thalo_binding",
+      "dependencies": [
+        "<!(node -p \"require('node-addon-api').targets\"):node_addon_api_except",
+      ],
+      "include_dirs": [
+        "src",
+      ],
+      "sources": [
+        "bindings/node/binding.cc",
+        "src/parser.c",
+      ],
+      "variables": {
+        "has_scanner": "<!(node -p \"fs.existsSync('src/scanner.c')\")"
+      },
+      "conditions": [
+        ["has_scanner=='true'", {
+          "sources+": ["src/scanner.c"],
+        }],
+        ["OS!='win'", {
+          "cflags_c": [
+            "-std=c11",
+          ],
+        }, { # OS == "win"
+          "cflags_c": [
+            "/std:c11",
+            "/utf-8",
+          ],
+        }],
+      ],
+    }
+  ]
+}

package/bindings/node/binding.cc

@@ -0,0 +1,19 @@
+#include <napi.h>
+
+typedef struct TSLanguage TSLanguage;
+
+extern "C" TSLanguage *tree_sitter_thalo();
+
+// "tree-sitter", "language" hashed with BLAKE2
+const napi_type_tag LANGUAGE_TYPE_TAG = {
+    0x8AF2E5212AD58ABF, 0xD5006CAD83ABBA16
+};
+
+Napi::Object Init(Napi::Env env, Napi::Object exports) {
+    auto language = Napi::External<TSLanguage>::New(env, tree_sitter_thalo());
+    language.TypeTag(&LANGUAGE_TYPE_TAG);
+    exports["language"] = language;
+    return exports;
+}
+
+NODE_API_MODULE(tree_sitter_thalo_binding, Init)

package/bindings/node/binding_test.js

@@ -0,0 +1,11 @@
+import assert from "node:assert";
+import { test } from "node:test";
+import Parser from "tree-sitter";
+
+test("can load grammar", () => {
+  const parser = new Parser();
+  assert.doesNotReject(async () => {
+    const { default: language } = await import("./index.js");
+    parser.setLanguage(language);
+  });
+});

package/bindings/node/index.d.ts

@@ -0,0 +1,60 @@
+type BaseNode = {
+  type: string;
+  named: boolean;
+};
+
+type ChildNode = {
+  multiple: boolean;
+  required: boolean;
+  types: BaseNode[];
+};
+
+type NodeInfo =
+  | (BaseNode & {
+      subtypes: BaseNode[];
+    })
+  | (BaseNode & {
+      fields: { [name: string]: ChildNode };
+      children: ChildNode[];
+    });
+
+/**
+ * The tree-sitter language object for this grammar.
+ *
+ * @see {@linkcode https://tree-sitter.github.io/node-tree-sitter/interfaces/Parser.Language.html Parser.Language}
+ *
+ * @example
+ * import Parser from "tree-sitter";
+ * import Thalo from "tree-sitter-thalo";
+ *
+ * const parser = new Parser();
+ * parser.setLanguage(Thalo);
+ */
+declare const binding: {
+  /**
+   * The inner language object.
+   * @private
+   */
+  language: unknown;
+
+  /**
+   * The content of the `node-types.json` file for this grammar.
+   *
+   * @see {@linkplain https://tree-sitter.github.io/tree-sitter/using-parsers/6-static-node-types Static Node Types}
+   */
+  nodeTypeInfo: NodeInfo[];
+
+  /** The syntax highlighting query for this grammar. */
+  HIGHLIGHTS_QUERY?: string;
+
+  /** The language injection query for this grammar. */
+  INJECTIONS_QUERY?: string;
+
+  /** The local variable query for this grammar. */
+  LOCALS_QUERY?: string;
+
+  /** The symbol tagging query for this grammar. */
+  TAGS_QUERY?: string;
+};
+
+export default binding;

package/bindings/node/index.js

@@ -0,0 +1,38 @@
+import { readFileSync } from "node:fs";
+import { fileURLToPath } from "node:url";
+
+const root = fileURLToPath(new URL("../..", import.meta.url));
+
+const binding =
+  typeof process.versions.bun === "string"
+    ? // Support `bun build --compile` by being statically analyzable enough to find the .node file at build-time
+      await import(`${root}/prebuilds/${process.platform}-${process.arch}/tree-sitter-thalo.node`)
+    : (await import("node-gyp-build")).default(root);
+
+try {
+  const nodeTypes = await import(`${root}/src/node-types.json`, { with: { type: "json" } });
+  binding.nodeTypeInfo = nodeTypes.default;
+} catch {}
+
+const queries = [
+  ["HIGHLIGHTS_QUERY", `${root}/queries/highlights.scm`],
+  ["INJECTIONS_QUERY", `${root}/queries/injections.scm`],
+  ["LOCALS_QUERY", `${root}/queries/locals.scm`],
+  ["TAGS_QUERY", `${root}/queries/tags.scm`],
+];
+
+for (const [prop, path] of queries) {
+  Object.defineProperty(binding, prop, {
+    configurable: true,
+    enumerable: true,
+    get() {
+      delete binding[prop];
+      try {
+        binding[prop] = readFileSync(path, "utf8");
+      } catch {}
+      return binding[prop];
+    },
+  });
+}
+
+export default binding;