ibex 0.1.0 → 0.2.0

data/schema/public-performance-comparison-v1.schema.json

@@ -0,0 +1,506 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://raw.githubusercontent.com/ydah/ibex/main/schema/public-performance-comparison-v1.schema.json",
+  "title": "Public grammar Ibex and Racc performance comparison v1",
+  "type": "object",
+  "additionalProperties": false,
+  "required": [
+    "artifact",
+    "schema_version",
+    "evidence_kind",
+    "recorded_at",
+    "third_party_code_warning",
+    "environment",
+    "configuration",
+    "manifest",
+    "projects"
+  ],
+  "properties": {
+    "artifact": { "const": "ibex_racc_public_performance_comparison" },
+    "schema_version": { "const": 1 },
+    "evidence_kind": { "enum": ["formal", "diagnostic_smoke"] },
+    "recorded_at": { "type": "string", "format": "date-time" },
+    "third_party_code_warning": { "type": "string", "minLength": 1 },
+    "environment": {
+      "type": "object",
+      "required": [
+        "git_revision",
+        "git_dirty",
+        "ruby_engine",
+        "ruby_version",
+        "ruby_platform",
+        "kernel_release",
+        "processors",
+        "yjit_enabled",
+        "ibex_version",
+        "racc_version",
+        "rubyopt"
+      ]
+    },
+    "configuration": { "$ref": "#/$defs/configuration" },
+    "manifest": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["path", "sha256"],
+      "properties": {
+        "path": { "const": "benchmark/public_workloads.json" },
+        "sha256": { "$ref": "#/$defs/sha256" }
+      }
+    },
+    "projects": {
+      "type": "array",
+      "minItems": 1,
+      "items": { "$ref": "#/$defs/project" }
+    }
+  },
+  "allOf": [
+    {
+      "if": {
+        "properties": {
+          "evidence_kind": { "const": "formal" }
+        },
+        "required": ["evidence_kind"]
+      },
+      "then": {
+        "properties": {
+          "environment": {
+            "properties": {
+              "git_dirty": { "const": false },
+              "git_tracked_dirty": { "const": false },
+              "git_untracked_dirty": { "const": false }
+            },
+            "required": ["git_dirty", "git_tracked_dirty", "git_untracked_dirty"]
+          },
+          "configuration": {
+            "properties": {
+              "runs": { "minimum": 10 },
+              "expected_racc_backend": { "const": "ruby" },
+              "allow_dirty_checkouts": { "const": false }
+            }
+          },
+          "projects": {
+            "items": { "$ref": "#/$defs/formalProject" }
+          }
+        }
+      }
+    }
+  ],
+  "$defs": {
+    "sha256": {
+      "type": "string",
+      "pattern": "^[0-9a-f]{64}$"
+    },
+    "sha1": {
+      "type": "string",
+      "pattern": "^[0-9a-f]{40}$"
+    },
+    "positiveNumber": {
+      "type": "number",
+      "exclusiveMinimum": 0
+    },
+    "nonNegativeNumber": {
+      "type": "number",
+      "minimum": 0
+    },
+    "stringArray": {
+      "type": "array",
+      "items": { "type": "string" }
+    },
+    "configuration": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "runs",
+        "warmup_workloads",
+        "measured_workloads",
+        "probe_workloads",
+        "bootstrap_samples",
+        "expected_racc_backend",
+        "process_order",
+        "runtime_scope",
+        "parser_lifecycles",
+        "allow_dirty_checkouts"
+      ],
+      "properties": {
+        "runs": { "type": "integer", "minimum": 1 },
+        "warmup_workloads": { "type": "integer", "minimum": 0 },
+        "measured_workloads": { "type": "integer", "minimum": 1 },
+        "probe_workloads": { "type": "integer", "minimum": 1, "maximum": 100 },
+        "bootstrap_samples": { "type": "integer", "minimum": 1000 },
+        "expected_racc_backend": { "enum": ["native", "ruby"] },
+        "process_order": { "const": "alternating_per_run" },
+        "runtime_scope": { "const": "end_to_end_lexer_inclusive" },
+        "parser_lifecycles": {
+          "type": "array",
+          "prefixItems": [
+            { "const": "reuse" },
+            { "const": "new_instance" }
+          ],
+          "items": false
+        },
+        "allow_dirty_checkouts": { "type": "boolean" }
+      }
+    },
+    "formalProject": {
+      "type": "object",
+      "required": ["repository", "scenarios"],
+      "properties": {
+        "repository": {
+          "type": "object",
+          "required": ["dirty", "tracked_dirty", "untracked_dirty"],
+          "properties": {
+            "dirty": { "const": false },
+            "tracked_dirty": { "const": false },
+            "untracked_dirty": { "const": false }
+          }
+        },
+        "scenarios": {
+          "type": "object",
+          "required": ["cold_generation", "warm_runtime_reuse", "warm_runtime_new_instance"],
+          "properties": {
+            "cold_generation": { "$ref": "#/$defs/formalGenerationScenario" },
+            "warm_runtime_reuse": { "$ref": "#/$defs/formalRuntimeScenario" },
+            "warm_runtime_new_instance": { "$ref": "#/$defs/formalRuntimeScenario" }
+          }
+        }
+      }
+    },
+    "formalGenerationScenario": {
+      "type": "object",
+      "required": ["implementations"],
+      "properties": {
+        "implementations": {
+          "type": "object",
+          "required": ["ibex", "racc"],
+          "properties": {
+            "ibex": { "$ref": "#/$defs/formalGenerationImplementation" },
+            "racc": { "$ref": "#/$defs/formalGenerationImplementation" }
+          }
+        }
+      }
+    },
+    "formalGenerationImplementation": {
+      "type": "object",
+      "required": ["observations"],
+      "properties": {
+        "observations": {
+          "type": "object",
+          "required": ["elapsed_ms"],
+          "properties": {
+            "elapsed_ms": {
+              "type": "array",
+              "minItems": 10
+            }
+          }
+        }
+      }
+    },
+    "formalRuntimeScenario": {
+      "type": "object",
+      "required": ["implementations"],
+      "properties": {
+        "implementations": {
+          "type": "object",
+          "required": ["ibex", "racc"],
+          "properties": {
+            "ibex": { "$ref": "#/$defs/formalRuntimeImplementation" },
+            "racc": {
+              "allOf": [
+                { "$ref": "#/$defs/formalRuntimeImplementation" },
+                {
+                  "properties": {
+                    "runtime_backend": { "const": "ruby" }
+                  },
+                  "required": ["runtime_backend"]
+                }
+              ]
+            }
+          }
+        }
+      }
+    },
+    "formalRuntimeImplementation": {
+      "type": "object",
+      "required": ["observations"],
+      "properties": {
+        "observations": {
+          "type": "object",
+          "required": ["elapsed_ms_per_parse", "allocated_objects_per_parse"],
+          "properties": {
+            "elapsed_ms_per_parse": {
+              "type": "array",
+              "minItems": 10
+            },
+            "allocated_objects_per_parse": {
+              "type": "array",
+              "minItems": 10
+            }
+          }
+        }
+      }
+    },
+    "repository": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "url",
+        "origin",
+        "revision",
+        "expected_revision",
+        "grammar_path",
+        "dependency_definition_path",
+        "dirty",
+        "tracked_dirty",
+        "untracked_dirty",
+        "status_sha256",
+        "grammar_sha256",
+        "dependency_definition_sha256",
+        "library_tree_oid"
+      ],
+      "properties": {
+        "url": { "type": "string", "minLength": 1 },
+        "origin": { "type": "string", "minLength": 1 },
+        "revision": { "$ref": "#/$defs/sha1" },
+        "expected_revision": { "$ref": "#/$defs/sha1" },
+        "grammar_path": { "type": "string", "minLength": 1 },
+        "dependency_definition_path": { "type": "string", "minLength": 1 },
+        "dirty": { "type": "boolean" },
+        "tracked_dirty": { "type": "boolean" },
+        "untracked_dirty": { "type": "boolean" },
+        "status_sha256": { "$ref": "#/$defs/sha256" },
+        "grammar_sha256": { "$ref": "#/$defs/sha256" },
+        "dependency_definition_sha256": { "$ref": "#/$defs/sha256" },
+        "library_tree_oid": {
+          "type": "string",
+          "pattern": "^[0-9a-f]{40,64}$"
+        }
+      }
+    },
+    "workload": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["id", "driver", "input_count", "inputs_sha256"],
+      "properties": {
+        "id": { "type": "string", "minLength": 1 },
+        "driver": { "type": "string", "minLength": 1 },
+        "input_count": { "type": "integer", "minimum": 1 },
+        "inputs_sha256": { "$ref": "#/$defs/sha256" }
+      }
+    },
+    "commands": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["generation", "worker"],
+      "properties": {
+        "generation": {
+          "type": "object",
+          "additionalProperties": false,
+          "required": ["ibex", "racc"],
+          "properties": {
+            "ibex": { "$ref": "#/$defs/stringArray" },
+            "racc": { "$ref": "#/$defs/stringArray" }
+          }
+        },
+        "worker": { "$ref": "#/$defs/stringArray" }
+      }
+    },
+    "description": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["median", "mad", "minimum", "maximum"],
+      "properties": {
+        "median": { "$ref": "#/$defs/nonNegativeNumber" },
+        "mad": { "$ref": "#/$defs/nonNegativeNumber" },
+        "minimum": { "$ref": "#/$defs/nonNegativeNumber" },
+        "maximum": { "$ref": "#/$defs/nonNegativeNumber" }
+      }
+    },
+    "metricComparison": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["ibex", "racc", "ibex_to_racc_ratio", "bootstrap_95_percent", "target_met"],
+      "properties": {
+        "ibex": { "$ref": "#/$defs/description" },
+        "racc": { "$ref": "#/$defs/description" },
+        "ibex_to_racc_ratio": { "$ref": "#/$defs/nonNegativeNumber" },
+        "bootstrap_95_percent": {
+          "type": "object",
+          "additionalProperties": false,
+          "required": ["lower", "upper", "samples", "seed"],
+          "properties": {
+            "lower": { "$ref": "#/$defs/nonNegativeNumber" },
+            "upper": { "$ref": "#/$defs/nonNegativeNumber" },
+            "samples": { "type": "integer", "minimum": 1000 },
+            "seed": { "type": "integer", "minimum": 0 }
+          }
+        },
+        "target_met": { "type": "boolean" }
+      }
+    },
+    "sizeComparison": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["ibex_bytes", "racc_bytes", "ibex_to_racc_ratio"],
+      "properties": {
+        "ibex_bytes": { "type": "integer", "minimum": 1 },
+        "racc_bytes": { "type": "integer", "minimum": 1 },
+        "ibex_to_racc_ratio": { "$ref": "#/$defs/nonNegativeNumber" }
+      }
+    },
+    "generationImplementation": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["observations", "statistics", "generated_bytes"],
+      "properties": {
+        "observations": {
+          "type": "object",
+          "additionalProperties": false,
+          "required": ["elapsed_ms"],
+          "properties": {
+            "elapsed_ms": {
+              "type": "array",
+              "minItems": 1,
+              "items": { "$ref": "#/$defs/positiveNumber" }
+            }
+          }
+        },
+        "statistics": {
+          "type": "object",
+          "additionalProperties": false,
+          "required": ["elapsed_ms"],
+          "properties": {
+            "elapsed_ms": { "$ref": "#/$defs/description" }
+          }
+        },
+        "generated_bytes": { "type": "integer", "minimum": 1 }
+      }
+    },
+    "runtimeImplementation": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "observations",
+        "statistics",
+        "generated_bytes",
+        "result_sha256",
+        "result_sequence_sha256",
+        "result_sequence_length",
+        "runtime_backend"
+      ],
+      "properties": {
+        "observations": {
+          "type": "object",
+          "additionalProperties": false,
+          "required": ["elapsed_ms_per_parse", "allocated_objects_per_parse"],
+          "properties": {
+            "elapsed_ms_per_parse": {
+              "type": "array",
+              "minItems": 1,
+              "items": { "$ref": "#/$defs/positiveNumber" }
+            },
+            "allocated_objects_per_parse": {
+              "type": "array",
+              "minItems": 1,
+              "items": { "$ref": "#/$defs/nonNegativeNumber" }
+            }
+          }
+        },
+        "statistics": {
+          "type": "object",
+          "additionalProperties": false,
+          "required": ["elapsed_ms_per_parse", "allocated_objects_per_parse"],
+          "properties": {
+            "elapsed_ms_per_parse": { "$ref": "#/$defs/description" },
+            "allocated_objects_per_parse": { "$ref": "#/$defs/description" }
+          }
+        },
+        "generated_bytes": { "type": "integer", "minimum": 1 },
+        "result_sha256": { "$ref": "#/$defs/sha256" },
+        "result_sequence_sha256": { "$ref": "#/$defs/sha256" },
+        "result_sequence_length": { "type": "integer", "minimum": 1, "maximum": 100 },
+        "runtime_backend": { "enum": ["native", "ruby"] }
+      }
+    },
+    "generationScenario": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["implementations", "comparison"],
+      "properties": {
+        "implementations": {
+          "type": "object",
+          "additionalProperties": false,
+          "required": ["ibex", "racc"],
+          "properties": {
+            "ibex": { "$ref": "#/$defs/generationImplementation" },
+            "racc": { "$ref": "#/$defs/generationImplementation" }
+          }
+        },
+        "comparison": {
+          "type": "object",
+          "additionalProperties": false,
+          "required": ["elapsed_ms", "generated_bytes"],
+          "properties": {
+            "elapsed_ms": { "$ref": "#/$defs/metricComparison" },
+            "generated_bytes": { "$ref": "#/$defs/sizeComparison" }
+          }
+        }
+      }
+    },
+    "runtimeScenario": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["implementations", "comparison"],
+      "properties": {
+        "implementations": {
+          "type": "object",
+          "additionalProperties": false,
+          "required": ["ibex", "racc"],
+          "properties": {
+            "ibex": { "$ref": "#/$defs/runtimeImplementation" },
+            "racc": { "$ref": "#/$defs/runtimeImplementation" }
+          }
+        },
+        "comparison": {
+          "type": "object",
+          "additionalProperties": false,
+          "required": [
+            "elapsed_ms_per_parse",
+            "allocated_objects_per_parse",
+            "generated_bytes",
+            "result_equivalent",
+            "result_sequence_equivalent"
+          ],
+          "properties": {
+            "elapsed_ms_per_parse": { "$ref": "#/$defs/metricComparison" },
+            "allocated_objects_per_parse": { "$ref": "#/$defs/metricComparison" },
+            "generated_bytes": { "$ref": "#/$defs/sizeComparison" },
+            "result_equivalent": { "const": true },
+            "result_sequence_equivalent": { "const": true }
+          }
+        }
+      }
+    },
+    "scenarios": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["cold_generation", "warm_runtime_reuse", "warm_runtime_new_instance"],
+      "properties": {
+        "cold_generation": { "$ref": "#/$defs/generationScenario" },
+        "warm_runtime_reuse": { "$ref": "#/$defs/runtimeScenario" },
+        "warm_runtime_new_instance": { "$ref": "#/$defs/runtimeScenario" }
+      }
+    },
+    "project": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["id", "repository", "workload", "commands", "scenarios"],
+      "properties": {
+        "id": { "enum": ["namae", "bcdice_command", "nokogiri_css"] },
+        "repository": { "$ref": "#/$defs/repository" },
+        "workload": { "$ref": "#/$defs/workload" },
+        "commands": { "$ref": "#/$defs/commands" },
+        "scenarios": { "$ref": "#/$defs/scenarios" }
+      }
+    }
+  }
+}