ibex 0.1.0 → 0.2.0

data/schema/lexer-ir-v1.schema.json

@@ -0,0 +1,215 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://raw.githubusercontent.com/ydah/ibex/main/schema/lexer-ir-v1.schema.json",
+  "title": "Ibex Lexer IR version 1",
+  "description": "A deterministic stateful lexer using longest-match and declaration-order priority.",
+  "type": "object",
+  "required": [
+    "ibex_ir",
+    "schema_version",
+    "initial_state",
+    "states",
+    "rules",
+    "warnings",
+    "source_provenance"
+  ],
+  "properties": {
+    "ibex_ir": {
+      "const": "lexer"
+    },
+    "schema_version": {
+      "const": 1
+    },
+    "initial_state": {
+      "const": "INITIAL"
+    },
+    "states": {
+      "type": "array",
+      "minItems": 1,
+      "uniqueItems": true,
+      "prefixItems": [
+        {
+          "const": "INITIAL"
+        }
+      ],
+      "items": {
+        "type": "string",
+        "minLength": 1
+      }
+    },
+    "rules": {
+      "type": "array",
+      "minItems": 1,
+      "items": {
+        "$ref": "#/$defs/rule"
+      }
+    },
+    "warnings": {
+      "type": "array",
+      "items": {
+        "$ref": "#/$defs/warning"
+      }
+    },
+    "source_provenance": {
+      "oneOf": [
+        {
+          "type": "null"
+        },
+        {
+          "$ref": "#/$defs/source_provenance"
+        }
+      ]
+    }
+  },
+  "additionalProperties": false,
+  "$defs": {
+    "location": {
+      "type": "object",
+      "required": [
+        "file",
+        "line",
+        "column"
+      ],
+      "properties": {
+        "file": {
+          "type": "string"
+        },
+        "line": {
+          "type": "integer",
+          "minimum": 1
+        },
+        "column": {
+          "type": "integer",
+          "minimum": 1
+        }
+      },
+      "additionalProperties": false
+    },
+    "rule": {
+      "type": "object",
+      "required": [
+        "id",
+        "state",
+        "kind",
+        "token",
+        "pattern",
+        "pattern_kind",
+        "options",
+        "action",
+        "loc"
+      ],
+      "properties": {
+        "id": {
+          "type": "integer",
+          "minimum": 0
+        },
+        "state": {
+          "type": "string",
+          "minLength": 1
+        },
+        "kind": {
+          "enum": [
+            "token",
+            "skip",
+            "on"
+          ]
+        },
+        "token": {
+          "type": [
+            "string",
+            "null"
+          ]
+        },
+        "pattern": {
+          "type": "string",
+          "minLength": 1
+        },
+        "pattern_kind": {
+          "enum": [
+            "regexp",
+            "literal"
+          ]
+        },
+        "options": {
+          "type": "string",
+          "pattern": "^[imx]*$"
+        },
+        "action": {
+          "type": [
+            "string",
+            "null"
+          ]
+        },
+        "loc": {
+          "$ref": "#/$defs/location"
+        }
+      },
+      "additionalProperties": false
+    },
+    "warning": {
+      "type": "object",
+      "required": [
+        "type",
+        "rule",
+        "loc"
+      ],
+      "properties": {
+        "type": {
+          "const": "redos"
+        },
+        "rule": {
+          "type": "integer",
+          "minimum": 0
+        },
+        "loc": {
+          "$ref": "#/$defs/location"
+        }
+      },
+      "additionalProperties": false
+    },
+    "source_provenance": {
+      "type": "object",
+      "required": [
+        "file",
+        "root",
+        "byte_span"
+      ],
+      "properties": {
+        "file": {
+          "type": [
+            "string",
+            "null"
+          ]
+        },
+        "root": {
+          "type": [
+            "string",
+            "null"
+          ]
+        },
+        "byte_span": {
+          "type": [
+            "object",
+            "null"
+          ],
+          "required": [
+            "start",
+            "end"
+          ],
+          "properties": {
+            "start": {
+              "type": "integer",
+              "minimum": 0
+            },
+            "end": {
+              "type": "integer",
+              "minimum": 0
+            }
+          },
+          "additionalProperties": false
+        }
+      },
+      "additionalProperties": false
+    }
+  }
+}

data/schema/migration-check-v1.schema.json

@@ -0,0 +1,60 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://raw.githubusercontent.com/ydah/ibex/main/schema/migration-check-v1.schema.json",
+  "title": "Ibex racc migration check version 1",
+  "type": "object",
+  "additionalProperties": false,
+  "required": [
+    "ibex_migration_check",
+    "schema_version",
+    "compatible",
+    "file",
+    "class_name",
+    "findings"
+  ],
+  "properties": {
+    "ibex_migration_check": { "const": "racc" },
+    "schema_version": { "const": 1 },
+    "compatible": { "type": "boolean" },
+    "file": { "type": "string", "minLength": 1 },
+    "class_name": {
+      "oneOf": [
+        { "type": "string", "pattern": "^[A-Z][A-Za-z0-9_]*(?:::[A-Z][A-Za-z0-9_]*)*$" },
+        { "type": "null" }
+      ]
+    },
+    "findings": {
+      "type": "array",
+      "items": { "$ref": "#/$defs/finding" }
+    }
+  },
+  "$defs": {
+    "location": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["file", "line", "column"],
+      "properties": {
+        "file": { "type": "string" },
+        "line": { "type": "integer", "minimum": 1 },
+        "column": { "type": "integer", "minimum": 1 }
+      }
+    },
+    "finding": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["code", "severity", "message", "location", "suggestion"],
+      "properties": {
+        "code": { "type": "string", "pattern": "^racc\\.[a-z_]+$" },
+        "severity": { "enum": ["error", "warning", "info"] },
+        "message": { "type": "string", "minLength": 1 },
+        "location": {
+          "oneOf": [
+            { "$ref": "#/$defs/location" },
+            { "type": "null" }
+          ]
+        },
+        "suggestion": { "type": "string", "minLength": 1 }
+      }
+    }
+  }
+}

data/schema/performance-comparison-v1.schema.json

@@ -0,0 +1,395 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://raw.githubusercontent.com/ydah/ibex/main/schema/performance-comparison-v1.schema.json",
+  "title": "Ibex and Racc performance comparison v1",
+  "type": "object",
+  "additionalProperties": false,
+  "required": [
+    "artifact",
+    "schema_version",
+    "recorded_at",
+    "environment",
+    "configuration",
+    "workload",
+    "commands",
+    "scenarios"
+  ],
+  "properties": {
+    "artifact": { "const": "ibex_racc_performance_comparison" },
+    "schema_version": { "const": 1 },
+    "recorded_at": { "type": "string", "format": "date-time" },
+    "environment": { "$ref": "#/$defs/environment" },
+    "configuration": { "$ref": "#/$defs/configuration" },
+    "workload": { "$ref": "#/$defs/workload" },
+    "commands": { "$ref": "#/$defs/commands" },
+    "scenarios": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "cold_generation",
+        "warm_runtime_end_to_end_reuse",
+        "warm_runtime_end_to_end_new_instance",
+        "warm_runtime_tokens_reuse",
+        "warm_runtime_tokens_new_instance"
+      ],
+      "properties": {
+        "cold_generation": { "$ref": "#/$defs/generationScenario" },
+        "warm_runtime_end_to_end_reuse": { "$ref": "#/$defs/runtimeScenario" },
+        "warm_runtime_end_to_end_new_instance": { "$ref": "#/$defs/runtimeScenario" },
+        "warm_runtime_tokens_reuse": { "$ref": "#/$defs/runtimeScenario" },
+        "warm_runtime_tokens_new_instance": { "$ref": "#/$defs/runtimeScenario" }
+      }
+    }
+  },
+  "$defs": {
+    "nonNegativeNumber": { "type": "number", "minimum": 0 },
+    "positiveNumber": { "type": "number", "exclusiveMinimum": 0 },
+    "sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" },
+    "stringArray": {
+      "type": "array",
+      "items": { "type": "string" }
+    },
+    "positiveSamples": {
+      "type": "array",
+      "minItems": 10,
+      "items": { "$ref": "#/$defs/positiveNumber" }
+    },
+    "nonNegativeSamples": {
+      "type": "array",
+      "minItems": 10,
+      "items": { "$ref": "#/$defs/nonNegativeNumber" }
+    },
+    "description": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["median", "mad", "minimum", "maximum"],
+      "properties": {
+        "median": { "$ref": "#/$defs/nonNegativeNumber" },
+        "mad": { "$ref": "#/$defs/nonNegativeNumber" },
+        "minimum": { "$ref": "#/$defs/nonNegativeNumber" },
+        "maximum": { "$ref": "#/$defs/nonNegativeNumber" }
+      }
+    },
+    "metricComparison": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["ibex", "racc", "ibex_to_racc_ratio", "bootstrap_95_percent", "target_met"],
+      "properties": {
+        "ibex": { "$ref": "#/$defs/description" },
+        "racc": { "$ref": "#/$defs/description" },
+        "ibex_to_racc_ratio": { "$ref": "#/$defs/nonNegativeNumber" },
+        "bootstrap_95_percent": {
+          "type": "object",
+          "additionalProperties": false,
+          "required": ["lower", "upper", "samples", "seed"],
+          "properties": {
+            "lower": { "$ref": "#/$defs/nonNegativeNumber" },
+            "upper": { "$ref": "#/$defs/nonNegativeNumber" },
+            "samples": { "type": "integer", "minimum": 1000 },
+            "seed": { "type": "integer", "minimum": 0 }
+          }
+        },
+        "target_met": { "type": "boolean" }
+      }
+    },
+    "generated": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["bytes"],
+      "properties": {
+        "bytes": { "type": "integer", "minimum": 1 }
+      }
+    },
+    "sizeComparison": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["ibex_bytes", "racc_bytes", "ibex_to_racc_ratio"],
+      "properties": {
+        "ibex_bytes": { "type": "integer", "minimum": 1 },
+        "racc_bytes": { "type": "integer", "minimum": 1 },
+        "ibex_to_racc_ratio": { "$ref": "#/$defs/nonNegativeNumber" }
+      }
+    },
+    "generationImplementation": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["observations", "statistics", "generated"],
+      "properties": {
+        "observations": {
+          "type": "object",
+          "additionalProperties": false,
+          "required": ["elapsed_ms"],
+          "properties": {
+            "elapsed_ms": { "$ref": "#/$defs/positiveSamples" }
+          }
+        },
+        "statistics": {
+          "type": "object",
+          "additionalProperties": false,
+          "required": ["elapsed_ms"],
+          "properties": {
+            "elapsed_ms": { "$ref": "#/$defs/description" }
+          }
+        },
+        "generated": { "$ref": "#/$defs/generated" }
+      }
+    },
+    "runtimeImplementation": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "observations",
+        "statistics",
+        "generated",
+        "result_sha256",
+        "behavior_sha256",
+        "result_sequence_sha256",
+        "result_sequence_length",
+        "runtime_backend"
+      ],
+      "properties": {
+        "observations": {
+          "type": "object",
+          "additionalProperties": false,
+          "required": ["elapsed_ms_per_parse", "allocated_objects_per_parse"],
+          "properties": {
+            "elapsed_ms_per_parse": { "$ref": "#/$defs/positiveSamples" },
+            "allocated_objects_per_parse": { "$ref": "#/$defs/nonNegativeSamples" }
+          }
+        },
+        "statistics": {
+          "type": "object",
+          "additionalProperties": false,
+          "required": ["elapsed_ms_per_parse", "allocated_objects_per_parse"],
+          "properties": {
+            "elapsed_ms_per_parse": { "$ref": "#/$defs/description" },
+            "allocated_objects_per_parse": { "$ref": "#/$defs/description" }
+          }
+        },
+        "generated": { "$ref": "#/$defs/generated" },
+        "result_sha256": { "$ref": "#/$defs/sha256" },
+        "behavior_sha256": { "$ref": "#/$defs/sha256" },
+        "result_sequence_sha256": { "$ref": "#/$defs/sha256" },
+        "result_sequence_length": { "type": "integer", "minimum": 1, "maximum": 100 },
+        "runtime_backend": { "enum": ["native", "ruby"] }
+      }
+    },
+    "generationScenario": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["implementations", "comparison"],
+      "properties": {
+        "implementations": {
+          "type": "object",
+          "additionalProperties": false,
+          "required": ["ibex", "racc"],
+          "properties": {
+            "ibex": { "$ref": "#/$defs/generationImplementation" },
+            "racc": { "$ref": "#/$defs/generationImplementation" }
+          }
+        },
+        "comparison": {
+          "type": "object",
+          "additionalProperties": false,
+          "required": ["elapsed_ms", "generated_bytes"],
+          "properties": {
+            "elapsed_ms": { "$ref": "#/$defs/metricComparison" },
+            "generated_bytes": { "$ref": "#/$defs/sizeComparison" }
+          }
+        }
+      }
+    },
+    "runtimeScenario": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["implementations", "comparison"],
+      "properties": {
+        "implementations": {
+          "type": "object",
+          "additionalProperties": false,
+          "required": ["ibex", "racc"],
+          "properties": {
+            "ibex": { "$ref": "#/$defs/runtimeImplementation" },
+            "racc": { "$ref": "#/$defs/runtimeImplementation" }
+          }
+        },
+        "comparison": {
+          "type": "object",
+          "additionalProperties": false,
+          "required": [
+            "elapsed_ms_per_parse",
+            "allocated_objects_per_parse",
+            "generated_bytes",
+            "result_equivalent",
+            "behavior_equivalent",
+            "result_sequence_equivalent"
+          ],
+          "properties": {
+            "elapsed_ms_per_parse": { "$ref": "#/$defs/metricComparison" },
+            "allocated_objects_per_parse": { "$ref": "#/$defs/metricComparison" },
+            "generated_bytes": { "$ref": "#/$defs/sizeComparison" },
+            "result_equivalent": { "const": true },
+            "behavior_equivalent": { "const": true },
+            "result_sequence_equivalent": { "const": true }
+          }
+        }
+      }
+    },
+    "environment": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "git_revision",
+        "git_dirty",
+        "git_tracked_dirty",
+        "git_untracked_dirty",
+        "ruby_engine",
+        "ruby_version",
+        "ruby_platform",
+        "ruby_description",
+        "ruby_executable",
+        "host_os",
+        "host_cpu",
+        "kernel_release",
+        "cpu_model",
+        "processors",
+        "yjit_available",
+        "yjit_enabled",
+        "ibex_version",
+        "racc_version",
+        "racc_executable",
+        "rubyopt"
+      ],
+      "properties": {
+        "git_revision": { "type": "string", "pattern": "^[0-9a-f]{40}$" },
+        "git_dirty": { "type": "boolean" },
+        "git_tracked_dirty": { "type": "boolean" },
+        "git_untracked_dirty": { "type": "boolean" },
+        "ruby_engine": { "type": "string", "minLength": 1 },
+        "ruby_version": { "type": "string", "minLength": 1 },
+        "ruby_platform": { "type": "string", "minLength": 1 },
+        "ruby_description": { "type": "string", "minLength": 1 },
+        "ruby_executable": { "type": "string", "minLength": 1 },
+        "host_os": { "type": "string", "minLength": 1 },
+        "host_cpu": { "type": "string", "minLength": 1 },
+        "kernel_release": { "type": "string", "minLength": 1 },
+        "cpu_model": {
+          "anyOf": [
+            { "type": "string", "minLength": 1 },
+            { "type": "null" }
+          ]
+        },
+        "processors": { "type": "integer", "minimum": 1 },
+        "yjit_available": { "type": "boolean" },
+        "yjit_enabled": { "type": "boolean" },
+        "ibex_version": { "type": "string", "minLength": 1 },
+        "racc_version": { "type": "string", "minLength": 1 },
+        "racc_executable": { "type": "string", "minLength": 1 },
+        "rubyopt": {
+          "type": "object",
+          "additionalProperties": false,
+          "required": ["present", "bytes", "sha256", "sanitized"],
+          "properties": {
+            "present": { "type": "boolean" },
+            "bytes": { "type": "integer", "minimum": 0 },
+            "sha256": {
+              "anyOf": [
+                { "$ref": "#/$defs/sha256" },
+                { "type": "null" }
+              ]
+            },
+            "sanitized": {
+              "type": "array",
+              "items": { "type": "string" }
+            }
+          }
+        }
+      }
+    },
+    "configuration": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "runs",
+        "warmup",
+        "runtime_iterations",
+        "workload_seed",
+        "behavior_probe_iterations",
+        "bootstrap_samples",
+        "expected_racc_backend",
+        "confidence_level",
+        "process_order"
+      ],
+      "properties": {
+        "runs": { "type": "integer", "minimum": 10 },
+        "warmup": { "type": "integer", "minimum": 0 },
+        "runtime_iterations": { "type": "integer", "minimum": 1 },
+        "workload_seed": { "type": "integer", "minimum": 0 },
+        "behavior_probe_iterations": { "type": "integer", "minimum": 1, "maximum": 100 },
+        "bootstrap_samples": { "type": "integer", "minimum": 1000 },
+        "expected_racc_backend": { "enum": ["native", "ruby"] },
+        "confidence_level": { "const": 0.95 },
+        "process_order": { "const": "alternating_per_run" }
+      }
+    },
+    "workload": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "grammar",
+        "input",
+        "source_grammar_sha256",
+        "comparison_grammar_sha256",
+        "grammar_adaptation",
+        "input_sha256",
+        "effective_input_sha256",
+        "parser_class",
+        "runtime_dimensions",
+        "parser_lifecycles"
+      ],
+      "properties": {
+        "grammar": { "const": "benchmark/grammars/representative.y" },
+        "input": { "const": "benchmark/grammars/representative.input" },
+        "source_grammar_sha256": { "$ref": "#/$defs/sha256" },
+        "comparison_grammar_sha256": { "$ref": "#/$defs/sha256" },
+        "grammar_adaptation": { "const": "normalize_eof_and_add_pretokenized_driver" },
+        "input_sha256": { "$ref": "#/$defs/sha256" },
+        "effective_input_sha256": { "$ref": "#/$defs/sha256" },
+        "parser_class": { "const": "BenchmarkRepresentativeParser" },
+        "runtime_dimensions": {
+          "type": "array",
+          "prefixItems": [
+            { "const": "lexer_inclusive" },
+            { "const": "pretokenized" }
+          ],
+          "items": false
+        },
+        "parser_lifecycles": {
+          "type": "array",
+          "prefixItems": [
+            { "const": "reuse" },
+            { "const": "new_instance" }
+          ],
+          "items": false
+        }
+      }
+    },
+    "commands": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["worker", "generation"],
+      "properties": {
+        "worker": { "$ref": "#/$defs/stringArray" },
+        "generation": {
+          "type": "object",
+          "additionalProperties": false,
+          "required": ["ibex", "racc"],
+          "properties": {
+            "ibex": { "$ref": "#/$defs/stringArray" },
+            "racc": { "$ref": "#/$defs/stringArray" }
+          }
+        }
+      }
+    }
+  }
+}