hullproof 0.0.0-stage → 0.5.0

package/packs/python/answer-key.json

@@ -0,0 +1,195 @@
+{
+  "pack": "python",
+  "fail_dir": "test/fixtures/map/fail",
+  "pass_dir": "test/fixtures/map/pass",
+  "stacks": [
+    "flask",
+    "fastapi",
+    "django"
+  ],
+  "detect_expected": [
+    "flask",
+    "fastapi",
+    "django",
+    "python"
+  ],
+  "mapped_what": "entry points",
+  "mapped": [
+    "http_route ANY /api/export",
+    "http_route ANY /api/items",
+    "http_route DELETE /api/notes/:note_id",
+    "http_route DELETE /api/notes/:note_id",
+    "http_route GET /api/export",
+    "http_route GET /api/export",
+    "http_route GET /api/items",
+    "http_route GET /api/items",
+    "http_route POST /api/admin/role",
+    "http_route POST /api/admin/role",
+    "http_route POST /api/admin/users/:uid/role",
+    "http_route POST /api/import",
+    "http_route POST /api/import",
+    "http_route POST /api/import",
+    "http_route POST /api/notes",
+    "http_route POST /api/notes",
+    "http_route POST /api/notes",
+    "http_route POST /api/notes/:pk/delete",
+    "http_route POST /api/upload",
+    "http_route POST /api/upload",
+    "http_route POST /api/upload",
+    "http_route POST /login",
+    "http_route POST /login",
+    "http_route POST /login",
+    "webhook POST /webhooks/pay",
+    "webhook POST /webhooks/pay",
+    "webhook POST /webhooks/pay"
+  ],
+  "flaws": [
+    {
+      "check": "admin-route-without-role",
+      "stack": "django"
+    },
+    {
+      "check": "admin-route-without-role",
+      "stack": "fastapi"
+    },
+    {
+      "check": "admin-route-without-role",
+      "stack": "flask"
+    },
+    {
+      "check": "data-read-without-auth",
+      "stack": "django"
+    },
+    {
+      "check": "data-read-without-auth",
+      "stack": "fastapi"
+    },
+    {
+      "check": "data-read-without-auth",
+      "stack": "flask"
+    },
+    {
+      "check": "mutating-route-without-auth",
+      "stack": "django"
+    },
+    {
+      "check": "mutating-route-without-auth",
+      "stack": "fastapi"
+    },
+    {
+      "check": "mutating-route-without-auth",
+      "stack": "flask"
+    },
+    {
+      "check": "privileged-action-without-audit",
+      "stack": "django"
+    },
+    {
+      "check": "privileged-action-without-audit",
+      "stack": "fastapi"
+    },
+    {
+      "check": "privileged-action-without-audit",
+      "stack": "flask"
+    },
+    {
+      "check": "privileged-field-from-request",
+      "stack": "django"
+    },
+    {
+      "check": "privileged-field-from-request",
+      "stack": "fastapi"
+    },
+    {
+      "check": "privileged-field-from-request",
+      "stack": "flask"
+    },
+    {
+      "check": "redirect-from-request",
+      "stack": "django"
+    },
+    {
+      "check": "redirect-from-request",
+      "stack": "fastapi"
+    },
+    {
+      "check": "redirect-from-request",
+      "stack": "flask"
+    },
+    {
+      "check": "request-url-fetched",
+      "stack": "django"
+    },
+    {
+      "check": "request-url-fetched",
+      "stack": "fastapi"
+    },
+    {
+      "check": "request-url-fetched",
+      "stack": "flask"
+    },
+    {
+      "check": "webhook-verify-before-use",
+      "stack": "django"
+    },
+    {
+      "check": "webhook-verify-before-use",
+      "stack": "fastapi"
+    },
+    {
+      "check": "webhook-verify-before-use",
+      "stack": "flask"
+    },
+    {
+      "check": "write-by-id-without-owner",
+      "stack": "django"
+    },
+    {
+      "check": "write-by-id-without-owner",
+      "stack": "fastapi"
+    },
+    {
+      "check": "write-by-id-without-owner",
+      "stack": "flask"
+    }
+  ],
+  "notes": [
+    "Measured on synthetic fixtures with a key kept in this repository. BenchmarkPython (Flask, 2463 routes) matched its decorator count in Phase 2, but no recall figure for planted flaws has been taken on a blind key yet."
+  ],
+  "recorded": [
+    {
+      "name": "fastapi/full-stack-fastapi-template (seeded sample of its route registrations)",
+      "kind": "third_party",
+      "blind_key": true,
+      "source": "bench/PHASE6C-MEASURED-2026-10-14.md, recall by bench/phase6c-sample.ts with seed 20261017, three byte identical ci runs",
+      "mapped": {
+        "what": "entry points (real route registrations)",
+        "expected": 23,
+        "matched": 23,
+        "extra": 0
+      },
+      "flaws": null,
+      "stability": {
+        "runs": 3,
+        "identical": true
+      }
+    },
+    {
+      "name": "healthchecks/healthchecks (seeded sample of its route registrations)",
+      "kind": "third_party",
+      "blind_key": true,
+      "source": "bench/PHASE6C-MEASURED-2026-10-14.md, recall by bench/phase6c-sample.ts with seed 20261017, three byte identical ci runs",
+      "mapped": {
+        "what": "entry points (real route registrations)",
+        "expected": 25,
+        "matched": 25,
+        "extra": 0
+      },
+      "flaws": null,
+      "stability": {
+        "runs": 3,
+        "identical": true
+      }
+    }
+  ]
+}

package/packs/python/fixture-result.json

@@ -0,0 +1,119 @@
+{
+  "detect": {
+    "expected": [
+      "django",
+      "fastapi",
+      "flask",
+      "python"
+    ],
+    "found": [
+      "django",
+      "django-auth",
+      "fastapi",
+      "flask",
+      "python",
+      "sqlite"
+    ]
+  },
+  "engine": "0.5.0",
+  "measured": "2026-10-14",
+  "measurements": [
+    {
+      "blind_key": false,
+      "flaws": {
+        "false_findings": 0,
+        "found": 27,
+        "planted": 27
+      },
+      "kind": "synthetic",
+      "mapped": {
+        "expected": 27,
+        "extra": 0,
+        "matched": 27,
+        "what": "entry points"
+      },
+      "name": "test/fixtures/map/fail and test/fixtures/map/pass",
+      "source": "node bench/packs-measure.ts python (key packs/python/answer-key.json)",
+      "stability": {
+        "identical": true,
+        "runs": 3
+      }
+    },
+    {
+      "blind_key": true,
+      "flaws": {
+        "false_findings": 0,
+        "found": 4,
+        "planted": 4
+      },
+      "kind": "synthetic",
+      "mapped": {
+        "expected": 8,
+        "extra": 0,
+        "matched": 8,
+        "what": "entry points"
+      },
+      "name": "test/fixtures/prove/flask-sqlite/vulnerable and test/fixtures/prove/flask-sqlite/fixed",
+      "source": "node bench/packs-measure.ts python (key kept outside the repository)",
+      "stability": {
+        "identical": true,
+        "runs": 3
+      }
+    },
+    {
+      "blind_key": true,
+      "flaws": null,
+      "kind": "third_party",
+      "mapped": {
+        "expected": 23,
+        "extra": 0,
+        "matched": 23,
+        "what": "entry points (real route registrations)"
+      },
+      "name": "fastapi/full-stack-fastapi-template (seeded sample of its route registrations)",
+      "source": "bench/PHASE6C-MEASURED-2026-10-14.md, recall by bench/phase6c-sample.ts with seed 20261017, three byte identical ci runs",
+      "stability": {
+        "identical": true,
+        "runs": 3
+      }
+    },
+    {
+      "blind_key": true,
+      "flaws": null,
+      "kind": "third_party",
+      "mapped": {
+        "expected": 25,
+        "extra": 0,
+        "matched": 25,
+        "what": "entry points (real route registrations)"
+      },
+      "name": "healthchecks/healthchecks (seeded sample of its route registrations)",
+      "source": "bench/PHASE6C-MEASURED-2026-10-14.md, recall by bench/phase6c-sample.ts with seed 20261017, three byte identical ci runs",
+      "stability": {
+        "identical": true,
+        "runs": 3
+      }
+    }
+  ],
+  "notes": [
+    "Measured on synthetic fixtures with a key kept in this repository. BenchmarkPython (Flask, 2463 routes) matched its decorator count in Phase 2, but no recall figure for planted flaws has been taken on a blind key yet.",
+    "Flask with SQLite: four planted authorization flaws and a fixed copy. The key was written before the first measurement.",
+    "Correction after the first measurement: the first flaw listed a second check by mistake (a copy of the delete flaw). Nothing else in the key changed.",
+    "First measurement on this fixture, before any engine change: 8 of 8 entry points, 3 of 4 flaws found, 6 findings on the fixed copy. The six were checks that did not see the decorators require_user and require_admin as a session read and a role check. The Python vocabulary now names require_* decorators; the fixed copy then gave 1 finding, a missing audit record on the admin route, which was a real gap in the fixture and was repaired in both copies. The measurement is therefore no longer blind for these two decorator names.",
+    "The read by id flaw is missed: check.authz.owner-filter-on-id-read does not find it in this layout (a Python SELECT with the id placeholder inside a function that is not in a file the check reads as a handler file).",
+    "2026-10-14: the first flaw also accepts check.map.read-by-id-without-owner, which was built in step 6c after these fixtures missed the flaw."
+  ],
+  "pack": "python",
+  "proof": [
+    {
+      "false_proofs_fixed": 0,
+      "identical": true,
+      "planted": 4,
+      "proven": 4,
+      "runs": 3,
+      "scope": "Flask with SQLite",
+      "source": "bench/results/phase6-flask-metrics-container.json (Colima, 3 rounds, fixture test/fixtures/prove/flask-sqlite)"
+    }
+  ],
+  "schema": 1
+}

package/packs/python/pack.json

@@ -0,0 +1,121 @@
+{
+  "schema": 1,
+  "id": "python",
+  "name": "Python",
+  "summary": "Flask, FastAPI, Django and Django REST framework: entry points, protections, taint rules and checks. L4 for Flask with a local SQLite file.",
+  "edition": "free",
+  "languages": [
+    "python"
+  ],
+  "frameworks": [
+    "flask",
+    "fastapi",
+    "django",
+    "quart",
+    "sanic",
+    "starlette"
+  ],
+  "level": "L3",
+  "scopes": [
+    {
+      "where": "Flask with SQLite",
+      "level": "L4"
+    }
+  ],
+  "limits": [
+    "Flask, FastAPI and Django are read; other Python frameworks are not. Planted flaws were not measured on a real application. A view that dispatches by method to decorated functions shows no protection."
+  ],
+  "parts": {
+    "detectors": [
+      "python",
+      "flask",
+      "fastapi",
+      "django",
+      "quart",
+      "sanic",
+      "starlette"
+    ],
+    "extractors": [
+      {
+        "file": "src/map/py.ts"
+      },
+      {
+        "file": "src/map/py-django.ts"
+      }
+    ],
+    "auth_patterns": [
+      {
+        "file": "src/map/vocab.ts"
+      }
+    ],
+    "sources_sinks": [
+      {
+        "file": "rules/semgrep/taint-python.yaml"
+      }
+    ],
+    "rules": [
+      {
+        "file": "rules/semgrep/taint-python.yaml",
+        "tests": [
+          "test/fixtures/rules/taint-python.py"
+        ]
+      }
+    ],
+    "checks": [
+      {
+        "id": "check.map.admin-route-without-role"
+      },
+      {
+        "id": "check.map.data-read-without-auth"
+      },
+      {
+        "id": "check.map.mutating-route-without-auth"
+      },
+      {
+        "id": "check.map.privileged-action-without-audit"
+      },
+      {
+        "id": "check.map.privileged-field-from-request"
+      },
+      {
+        "id": "check.map.redirect-from-request"
+      },
+      {
+        "id": "check.map.request-url-fetched"
+      },
+      {
+        "id": "check.map.webhook-verify-before-use"
+      },
+      {
+        "id": "check.map.write-by-id-without-owner"
+      },
+      {
+        "id": "check.map.read-by-id-without-owner"
+      },
+      {
+        "id": "check.authz.admin-role-check"
+      },
+      {
+        "id": "check.authz.owner-filter-on-id-read"
+      }
+    ],
+    "mapping": [
+      {
+        "file": "src/checks/map/entry-checks.ts"
+      }
+    ],
+    "start_recipe": {
+      "file": "src/prove/recipe.ts",
+      "recipe": "python-flask"
+    }
+  },
+  "fixture": {
+    "dir": "test/fixtures/map/fail",
+    "fixed_dir": "test/fixtures/map/pass",
+    "answer_keys": [
+      "in_repo",
+      "outside_repo"
+    ],
+    "result": "packs/python/fixture-result.json"
+  }
+}

package/packs/rails/answer-key.json

@@ -0,0 +1,84 @@
+{
+  "pack": "rails",
+  "fail_dir": "test/fixtures/map-more/fail",
+  "pass_dir": "test/fixtures/map-more/pass",
+  "stacks": [
+    "rails"
+  ],
+  "detect_expected": [
+    "rails",
+    "ruby"
+  ],
+  "mapped_what": "entry points",
+  "mapped": [
+    "http_route DELETE /notes/:id",
+    "http_route GET /api/export",
+    "http_route GET /api/items",
+    "http_route POST /admin/users/:id/role",
+    "http_route POST /api/import",
+    "http_route POST /api/upload",
+    "http_route POST /login",
+    "http_route POST /notes",
+    "webhook POST /webhooks/pay"
+  ],
+  "flaws": [
+    {
+      "check": "admin-route-without-role",
+      "stack": "rails"
+    },
+    {
+      "check": "data-read-without-auth",
+      "stack": "rails"
+    },
+    {
+      "check": "mutating-route-without-auth",
+      "stack": "rails"
+    },
+    {
+      "check": "privileged-action-without-audit",
+      "stack": "rails"
+    },
+    {
+      "check": "privileged-field-from-request",
+      "stack": "rails"
+    },
+    {
+      "check": "redirect-from-request",
+      "stack": "rails"
+    },
+    {
+      "check": "request-url-fetched",
+      "stack": "rails"
+    },
+    {
+      "check": "webhook-verify-before-use",
+      "stack": "rails"
+    },
+    {
+      "check": "write-by-id-without-owner",
+      "stack": "rails"
+    }
+  ],
+  "notes": [
+    "Synthetic fixture only. No real Rails application has been measured."
+  ],
+  "recorded": [
+    {
+      "name": "lobsters/lobsters (seeded sample of its route registrations)",
+      "kind": "third_party",
+      "blind_key": true,
+      "source": "bench/PHASE6C-MEASURED-2026-10-14.md, recall by bench/phase6c-sample.ts with seed 20261017, three byte identical ci runs",
+      "mapped": {
+        "what": "entry points (real route registrations)",
+        "expected": 25,
+        "matched": 25,
+        "extra": 0
+      },
+      "flaws": null,
+      "stability": {
+        "runs": 3,
+        "identical": true
+      }
+    }
+  ]
+}

package/packs/rails/fixture-result.json

@@ -0,0 +1,60 @@
+{
+  "detect": {
+    "expected": [
+      "rails",
+      "ruby"
+    ],
+    "found": [
+      "rails",
+      "ruby"
+    ]
+  },
+  "engine": "0.5.0",
+  "measured": "2026-10-14",
+  "measurements": [
+    {
+      "blind_key": false,
+      "flaws": {
+        "false_findings": 0,
+        "found": 9,
+        "planted": 9
+      },
+      "kind": "synthetic",
+      "mapped": {
+        "expected": 9,
+        "extra": 0,
+        "matched": 9,
+        "what": "entry points"
+      },
+      "name": "test/fixtures/map-more/fail and test/fixtures/map-more/pass",
+      "source": "node bench/packs-measure.ts rails (key packs/rails/answer-key.json)",
+      "stability": {
+        "identical": true,
+        "runs": 3
+      }
+    },
+    {
+      "blind_key": true,
+      "flaws": null,
+      "kind": "third_party",
+      "mapped": {
+        "expected": 25,
+        "extra": 0,
+        "matched": 25,
+        "what": "entry points (real route registrations)"
+      },
+      "name": "lobsters/lobsters (seeded sample of its route registrations)",
+      "source": "bench/PHASE6C-MEASURED-2026-10-14.md, recall by bench/phase6c-sample.ts with seed 20261017, three byte identical ci runs",
+      "stability": {
+        "identical": true,
+        "runs": 3
+      }
+    }
+  ],
+  "notes": [
+    "Synthetic fixture only. No real Rails application has been measured."
+  ],
+  "pack": "rails",
+  "proof": [],
+  "schema": 1
+}

package/packs/rails/pack.json

@@ -0,0 +1,78 @@
+{
+  "schema": 1,
+  "id": "rails",
+  "name": "Ruby on Rails",
+  "summary": "config/routes.rb joined to controller actions and their before_action filters, with checks.",
+  "edition": "free",
+  "languages": [
+    "ruby"
+  ],
+  "frameworks": [
+    "rails"
+  ],
+  "level": "L2",
+  "limits": [
+    "Planted flaws were not measured on a real application. A filter that only looks a user up is told apart from one that blocks a guest by reading its body, which other shapes of filter may defeat."
+  ],
+  "parts": {
+    "detectors": [
+      "ruby",
+      "rails"
+    ],
+    "extractors": [
+      {
+        "file": "src/map/rails.ts"
+      }
+    ],
+    "auth_patterns": [
+      {
+        "file": "src/map/light-profiles.ts"
+      }
+    ],
+    "checks": [
+      {
+        "id": "check.map.admin-route-without-role"
+      },
+      {
+        "id": "check.map.data-read-without-auth"
+      },
+      {
+        "id": "check.map.mutating-route-without-auth"
+      },
+      {
+        "id": "check.map.privileged-action-without-audit"
+      },
+      {
+        "id": "check.map.privileged-field-from-request"
+      },
+      {
+        "id": "check.map.redirect-from-request"
+      },
+      {
+        "id": "check.map.request-url-fetched"
+      },
+      {
+        "id": "check.map.webhook-verify-before-use"
+      },
+      {
+        "id": "check.map.write-by-id-without-owner"
+      },
+      {
+        "id": "check.map.read-by-id-without-owner"
+      }
+    ],
+    "mapping": [
+      {
+        "file": "src/checks/map/entry-checks.ts"
+      }
+    ]
+  },
+  "fixture": {
+    "dir": "test/fixtures/map-more/fail",
+    "fixed_dir": "test/fixtures/map-more/pass",
+    "answer_keys": [
+      "in_repo"
+    ],
+    "result": "packs/rails/fixture-result.json"
+  }
+}

package/packs/rust/fixture-result.json

@@ -0,0 +1,43 @@
+{
+  "detect": {
+    "expected": [
+      "axum",
+      "rust"
+    ],
+    "found": [
+      "axum",
+      "rust"
+    ]
+  },
+  "engine": "0.5.0",
+  "measured": "2026-10-14",
+  "measurements": [
+    {
+      "blind_key": true,
+      "flaws": {
+        "false_findings": 0,
+        "found": 3,
+        "planted": 6
+      },
+      "kind": "synthetic",
+      "mapped": {
+        "expected": 10,
+        "extra": 0,
+        "matched": 10,
+        "what": "entry points"
+      },
+      "name": "test/fixtures/packs/rust/fail and test/fixtures/packs/rust/pass",
+      "source": "node bench/packs-measure.ts rust (key kept outside the repository)",
+      "stability": {
+        "identical": true,
+        "runs": 3
+      }
+    }
+  ],
+  "notes": [
+    "Rust service on axum: six planted flaws and a fixed copy. Two are settled by checks that belong to the Pro edition. The export route flaw needs a read of the handler body, which the Rust reader does not do. Written before the Rust extractor existed."
+  ],
+  "pack": "rust",
+  "proof": [],
+  "schema": 1
+}