@rejot-dev/tree-sitter-thalo 0.0.0

package/grammar.js

@@ -0,0 +1,303 @@
+export default grammar({
+  name: "thalo",
+
+  extras: (_) => [" "],
+
+  externals: ($) => [
+    $["_indent"], // Newline followed by indentation (1+ spaces or tab)
+    $["_content_blank"], // Blank line within content blocks
+    $["error_sentinel"], // Detects error recovery mode
+  ],
+
+  // Resolve ambiguity: link after directive could be argument or trailing link/tag
+  conflicts: ($) => [[$.data_entry]],
+
+  rules: {
+    source_file: ($) => repeat(choice($.entry, $.comment, $._nl)),
+
+    // Comment token (the text of a comment)
+    comment: (_) => token(seq("//", /[^\r\n]*/)),
+
+    // Indented comment line (within entries) - same prec as metadata (will be tried via choice)
+    comment_line: ($) => prec(2, seq($["_indent"], $.comment)),
+
+    // =========================================================================
+    // Unified entry structure
+    // Entry types are differentiated by the directive keyword in the header.
+    // Schema entries (define-entity, alter-entity) have schema blocks.
+    // Data entries (create, update, define-synthesis, actualize-synthesis) have metadata/content.
+    // =========================================================================
+
+    entry: ($) => choice($.schema_entry, $.data_entry),
+
+    // Schema entries: define-entity or alter-entity with optional schema blocks
+    schema_entry: ($) =>
+      seq(
+        field("timestamp", $.timestamp),
+        field("directive", $.schema_directive),
+        field("argument", $.identifier),
+        field("title", $.title),
+        repeat(choice($.link, $.tag)),
+        repeat($._schema_block),
+      ),
+
+    schema_directive: (_) => choice("define-entity", "alter-entity"),
+
+    // Data entries: instance, synthesis, or actualize with metadata and optional content
+    data_entry: ($) =>
+      seq(
+        field("timestamp", $.timestamp),
+        field("directive", $.data_directive),
+        field("argument", optional(choice($.identifier, $.link))),
+        field("title", optional($.title)),
+        repeat(choice($.link, $.tag)),
+        repeat(choice($.metadata, $.comment_line)),
+        optional($.content),
+      ),
+
+    data_directive: (_) => choice("create", "update", "define-synthesis", "actualize-synthesis"),
+
+    // Identifier for entity names (lore, opinion, etc.) and custom entity definitions
+    identifier: (_) => token(/[a-z][a-zA-Z0-9\-_]*/),
+
+    // prec(2) to prefer metadata over content_line when we see key:value
+    metadata: ($) => prec(2, seq($["_indent"], field("key", $.key), ":", field("value", $.value))),
+
+    // =========================================================================
+    // Schema blocks (# Metadata, # Sections, # Remove Metadata, # Remove Sections)
+    // =========================================================================
+
+    _schema_block: ($) =>
+      choice($.metadata_block, $.sections_block, $.remove_metadata_block, $.remove_sections_block),
+
+    metadata_block: ($) => prec(2, seq($._metadata_header, repeat1($.field_definition))),
+    sections_block: ($) => prec(2, seq($._sections_header, $._section_lines)),
+    remove_metadata_block: ($) => prec(2, seq($._remove_metadata_header, repeat1($.field_removal))),
+    remove_sections_block: ($) =>
+      prec(2, seq($._remove_sections_header, repeat1($.section_removal))),
+
+    _section_lines: ($) => prec.right(repeat1($.section_definition)),
+
+    // Block headers: newline + optional blank lines + indent + "# BlockName"
+    // Note: do NOT consume trailing spaces; they are ignored via `extras`.
+    _metadata_header: (_) => token(/\r?\n(?:[ \t]*\r?\n)*(?:\t|[ \t][ \t])+# Metadata/),
+    _sections_header: (_) => token(/\r?\n(?:[ \t]*\r?\n)*(?:\t|[ \t][ \t])+# Sections/),
+    _remove_metadata_header: (_) =>
+      token(/\r?\n(?:[ \t]*\r?\n)*(?:\t|[ \t][ \t])+# Remove Metadata/),
+    _remove_sections_header: (_) =>
+      token(/\r?\n(?:[ \t]*\r?\n)*(?:\t|[ \t][ \t])+# Remove Sections/),
+
+    // =========================================================================
+    // Field definitions (for schema metadata blocks)
+    // =========================================================================
+
+    field_definition: ($) =>
+      seq(
+        $._field_line_start,
+        optional($.optional_marker),
+        ":",
+        field("type", $.type_expression),
+        optional(seq("=", field("default", $.default_value))),
+        optional(seq(";", field("description", $.description))),
+      ),
+
+    // Newline + indent + field name (aliased to field_name in AST)
+    _field_line_start: ($) => alias($._field_name_token, $["field_name"]),
+    _field_name_token: (_) => token(/\r?\n(?:\t|[ \t][ \t])+[a-z][a-zA-Z0-9\-_]*/),
+
+    optional_marker: (_) => "?",
+    description: (_) => token(/"[^"]*"/),
+
+    // =========================================================================
+    // Section definitions (for schema sections blocks)
+    // =========================================================================
+
+    // Section names start uppercase, may contain spaces: "Key Takeaways"
+    section_definition: ($) =>
+      seq(
+        $._section_line_start,
+        optional($.optional_marker),
+        optional(seq(";", field("description", $.description))),
+      ),
+
+    // Newline + indent + section name (aliased to section_name in AST)
+    _section_line_start: ($) => alias($._section_name_token, $["section_name"]),
+    _section_name_token: (_) => token(/\r?\n(?:\t|[ \t][ \t])+[A-Z][a-zA-Z0-9]*( +[a-zA-Z0-9]+)*/),
+
+    // =========================================================================
+    // Removals (for alter-entity)
+    // =========================================================================
+
+    field_removal: ($) =>
+      seq($._field_line_start, optional(seq(";", field("reason", $.description)))),
+    section_removal: ($) =>
+      seq($._section_line_start, optional(seq(";", field("reason", $.description)))),
+
+    // =========================================================================
+    // Type expressions (for field definitions)
+    // =========================================================================
+
+    type_expression: ($) => choice($.union_type, $._type_term),
+    union_type: ($) => prec.left(1, seq($._type_term, repeat1(seq("|", $._type_term)))),
+    _type_term: ($) => choice($.array_type, $.primitive_type, $.literal_type, $.unknown_type),
+    array_type: ($) => seq($._array_element, token.immediate("[]")),
+    _array_element: ($) => choice($.primitive_type, $.literal_type, $.paren_type, $.unknown_type),
+    paren_type: ($) => seq("(", $.type_expression, ")"),
+    primitive_type: (_) => choice("string", "datetime", "daterange", "link", "number"),
+    literal_type: (_) => token(/"[^"]*"/),
+    // Fallback for unrecognized type identifiers (e.g., "date-time" typo)
+    // Tree-sitter prefers exact matches (primitive_type) over regex patterns
+    unknown_type: (_) => token(/[a-z][a-zA-Z0-9\-_]*/),
+    default_value: ($) => choice($.quoted_value, $.link, $.datetime_value, $.number_value),
+
+    // =========================================================================
+    // Content (markdown body for instance entries)
+    // =========================================================================
+
+    // Content must start with a markdown header (# Section Name)
+    content: ($) =>
+      prec.right(
+        seq(
+          repeat($["_content_blank"]),
+          $.markdown_header,
+          repeat(choice($.markdown_header, $.content_line, $.comment_line, $["_content_blank"])),
+        ),
+      ),
+
+    // prec(2) for headers vs prec(1) for content lines (headers win when line starts with #)
+    markdown_header: ($) => prec.right(2, seq($["_indent"], $.md_indicator, $.md_heading_text)),
+    content_line: ($) => prec.right(1, seq($["_indent"], $.content_text)),
+
+    md_indicator: (_) => token.immediate(/#+/),
+    md_heading_text: (_) => token.immediate(/ [^\r\n]+/),
+    // Must not start with # (would be header) or // (would be comment)
+    content_text: (_) => token.immediate(/[^#/\r\n][^\r\n]*|\/[^/\r\n][^\r\n]*/),
+
+    // =========================================================================
+    // Common tokens
+    // =========================================================================
+
+    _nl: (_) => /\r?\n/,
+
+    // Timestamp is decomposed into date, T separator, time, and optional timezone parts.
+    // Each part is a separate token using token.immediate() to prevent whitespace.
+    // Missing timezone is validated in builder.ts and produces a specific error.
+    timestamp: ($) =>
+      seq(
+        field("date", $.timestamp_date),
+        $.timestamp_t,
+        field("time", $.timestamp_time),
+        field("tz", optional($.timestamp_tz)),
+      ),
+    timestamp_date: (_) => token(/[12]\d{3}-[01]\d-[0-3]\d/),
+    timestamp_t: (_) => token.immediate("T"),
+    timestamp_time: (_) => token.immediate(/[0-2]\d:[0-5]\d/),
+    timestamp_tz: (_) => token.immediate(/Z|[+-][0-2]\d:[0-5]\d/),
+    // Allow unclosed quotes to terminate at newline for error recovery
+    title: (_) => token(/"[^"\r\n]*"?/),
+    link: (_) => token(/\^[A-Za-z0-9\-_/.:]*[A-Za-z0-9]/),
+    tag: (_) => token(/#[A-Za-z0-9\-_/.]+/),
+    key: (_) => token(/[a-z][a-zA-Z0-9\-_]*/),
+
+    // =========================================================================
+    // Typed metadata values
+    // =========================================================================
+
+    // Value parsing uses typed tokens. All values must be explicitly typed:
+    // - Links: ^identifier
+    // - Quoted strings: "text" (required for literal types like "fact")
+    // - Datetime: YYYY-MM-DD or YYYY-MM-DDTHH:MM (date with optional time)
+    // - Daterange: YYYY ~ YYYY, YYYY, YYYY-MM, YYYY Q1, etc.
+    // - Numbers: 123, -45.67
+    // - Queries: entity where conditions
+    // - Arrays: comma-separated values of any type
+    value: ($) =>
+      choice(
+        prec.dynamic(6, $.value_array), // Comma-separated values (2+ elements)
+        prec.dynamic(5, $.daterange), // Dateranges with ~, Q, or implicit period
+        prec.dynamic(4, $.datetime_value), // YYYY-MM-DD or YYYY-MM-DDTHH:MM
+        prec.dynamic(3, $.query), // entity where conditions
+        prec.dynamic(3, $.link), // ^identifier
+        prec.dynamic(3, $.quoted_value), // "quoted text"
+        prec.dynamic(2, $.number_value), // 123, -45.67
+      ),
+
+    // Quoted string as a value (required for literal types)
+    quoted_value: (_) => token(/"[^"]*"/),
+
+    // Number value: integer or float (e.g., 123, -45.67)
+    number_value: (_) => token(/-?\d+(\.\d+)?/),
+
+    // Datetime value: date with optional time (split into tokens)
+    // YYYY-MM-DD or YYYY-MM-DDTHH:MM or YYYY-MM-DDTHH:MMZ
+    datetime_value: ($) =>
+      seq(
+        field("date", $.datetime_date),
+        optional(
+          seq($.datetime_t, field("time", $.datetime_time), optional(field("tz", $.datetime_tz))),
+        ),
+      ),
+    datetime_date: (_) => token(/[12]\d{3}-[01]\d-[0-3]\d/),
+    datetime_t: (_) => token.immediate("T"),
+    datetime_time: (_) => token.immediate(/[0-2]\d:[0-5]\d/),
+    datetime_tz: (_) => token.immediate(/Z|[+-][0-2]\d:[0-5]\d/),
+
+    // Daterange: date period or range
+    // Formats:
+    //   a. Single period: YYYY-MM, YYYY Q1 (implicit start~end - YYYY alone conflicts with number)
+    //   b. Two full dates: YYYY-MM-DD ~ YYYY-MM-DD
+    //   c. Two partial: YYYY ~ YYYY, YYYY-MM ~ YYYY-MM
+    //   d. Open-ended: YYYY ~, YYYY-MM ~ (start till now)
+    // Note: Bare YYYY is ambiguous with number type, so use "YYYY ~" for year periods
+    daterange: (_) =>
+      choice(
+        // Quarter: YYYY Q1-Q4
+        token(/[12]\d{3} +Q[1-4]/),
+        // Range with ~: partial ~ optional(partial) - includes open-ended "YYYY ~"
+        token(/[12]\d{3}(-[01]\d(-[0-3]\d)?)? *~( *[12]\d{3}(-[01]\d(-[0-3]\d)?)?)?/),
+        // Implicit month: YYYY-MM (hyphen disambiguates from number)
+        token(/[12]\d{3}-[01]\d/),
+      ),
+
+    // Unified array: comma-separated values of any type
+    value_array: ($) =>
+      prec.right(seq($._value_array_element, repeat1(seq(",", $._value_array_element)))),
+
+    _value_array_element: ($) =>
+      choice($.link, $.quoted_value, $.datetime_value, $.daterange, $.query, $.number_value),
+
+    // =========================================================================
+    // Query expressions (for sources metadata)
+    // =========================================================================
+
+    // Single query: entity where conditions
+    query: ($) =>
+      seq(field("entity", $.query_entity), "where", field("conditions", $.query_conditions)),
+
+    // Query entity (lore, journal, opinion, reference)
+    query_entity: (_) => token(/[a-z][a-zA-Z0-9\-_]*/),
+
+    // Conditions joined by "and"
+    query_conditions: ($) => seq($.query_condition, repeat(seq("and", $.query_condition))),
+
+    // Individual condition types
+    query_condition: ($) =>
+      choice(
+        $.field_condition, // field = value
+        $.tag_condition, // #tag
+        $.link_condition, // ^link
+      ),
+
+    field_condition: ($) =>
+      seq(field("field", $.condition_field), "=", field("value", $._condition_value)),
+
+    // Condition field name
+    condition_field: (_) => token(/[a-z][a-zA-Z0-9\-_]*/),
+
+    // Condition values: links or quoted strings only (no plain values)
+    _condition_value: ($) => choice($.link, $.quoted_value),
+
+    tag_condition: ($) => $.tag,
+    link_condition: ($) => $.link,
+  },
+});

package/package.json

@@ -0,0 +1,45 @@
+{
+  "name": "@rejot-dev/tree-sitter-thalo",
+  "version": "0.0.0",
+  "type": "module",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/rejot-dev/thalo.git",
+    "directory": "packages/grammar"
+  },
+  "license": "MIT",
+  "main": "bindings/node/index.js",
+  "types": "bindings/node/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./bindings/node/index.d.ts",
+      "default": "./bindings/node/index.js"
+    },
+    "./tree-sitter-thalo.wasm": "./tree-sitter-thalo.wasm"
+  },
+  "files": [
+    "grammar.js",
+    "binding.gyp",
+    "prebuilds/**",
+    "bindings/node/*",
+    "queries/*",
+    "src/**",
+    "tree-sitter-thalo.wasm"
+  ],
+  "devDependencies": {
+    "@types/node": "^24",
+    "node-gyp": "^10.2.0",
+    "tree-sitter-cli": "^0.26.3",
+    "node-addon-api": "^8.5.0",
+    "node-gyp-build": "^4.8.4",
+    "@rejot-private/typescript-config": "0.0.1"
+  },
+  "scripts": {
+    "build": "tree-sitter generate && node scripts/check-rebuild.mjs --fix",
+    "build:wasm": "tree-sitter build --wasm --output tree-sitter-thalo.wasm",
+    "build:native": "pnpm exec node-gyp rebuild",
+    "check:native": "node scripts/check-rebuild.mjs",
+    "test": "tree-sitter test",
+    "types:check": "tsc --noEmit"
+  }
+}

package/queries/highlights.scm

@@ -0,0 +1,71 @@
+; Timestamps - displayed as special constants
+(timestamp) @number
+
+; Directives - keywords for entry types
+(data_directive) @keyword
+(schema_directive) @keyword
+
+; Entity identifier in schema definitions
+(identifier) @type
+
+; Title strings
+(title) @string
+
+; Links (^reference-id)
+(link) @tag
+
+; Tags (#category)
+(tag) @attribute
+
+; Metadata key-value pairs
+(metadata
+  (key) @property
+  (value) @string)
+
+; Field definitions in schema
+(field_definition
+  (field_name) @property
+  (optional_marker)? @punctuation.special
+  (type_expression) @type
+  (default_value)? @string
+  (description)? @comment)
+
+; Section definitions in schema
+; Using @variable instead of @label since @label isn't in most themes
+(section_definition
+  (section_name) @variable
+  (optional_marker)? @punctuation.special
+  (description)? @comment)
+
+; Type expressions
+(primitive_type) @type.builtin
+(literal_type) @string
+(array_type) @type
+
+; Punctuation
+":" @punctuation.delimiter
+";" @punctuation.delimiter
+"=" @operator
+"|" @operator
+"[]" @punctuation.bracket
+
+; Markdown headers in content
+(markdown_header) @markup.heading
+
+; Content lines (regular text)
+(content_line) @text
+
+; Field/section removals
+(field_removal
+  (field_name) @property
+  (description)? @comment)
+
+(section_removal
+  (section_name) @variable
+  (description)? @comment)
+
+; Description strings (in schema definitions)
+(description) @comment
+
+; Comments
+(comment) @comment