@infernetprotocol/liquid-agent 0.0.0-stage → 0.1.66

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. package/LICENSE +15 -0
  2. package/README.md +15 -2
  3. package/fixtures/contract.json +6 -0
  4. package/fixtures/invalid/episode-extra-field.json +275 -0
  5. package/fixtures/invalid/episode-no-capture.json +265 -0
  6. package/fixtures/invalid/episode-unknown-action.json +274 -0
  7. package/fixtures/invalid/model-manifest-bad-hash.json +92 -0
  8. package/fixtures/invalid/predict-negative-seq.json +48 -0
  9. package/fixtures/invalid/proposal-bad-abstain.json +31 -0
  10. package/fixtures/invalid/proposal-confidence-range.json +31 -0
  11. package/fixtures/invalid/session-bad-seq.json +17 -0
  12. package/fixtures/invalid/train-job-bad-family.json +44 -0
  13. package/fixtures/invalid/train-job-zero-lr.json +44 -0
  14. package/fixtures/valid/action-proposal.json +27 -0
  15. package/fixtures/valid/dataset-bundle.json +3747 -0
  16. package/fixtures/valid/dataset-manifest.json +126 -0
  17. package/fixtures/valid/episode.json +270 -0
  18. package/fixtures/valid/inference-session.json +13 -0
  19. package/fixtures/valid/model-manifest.json +88 -0
  20. package/fixtures/valid/predict-request.json +44 -0
  21. package/fixtures/valid/train-job.json +40 -0
  22. package/package.json +28 -3
  23. package/runtime/infernet_liquid/__init__.py +6 -0
  24. package/runtime/infernet_liquid/__main__.py +95 -0
  25. package/runtime/infernet_liquid/bench.py +125 -0
  26. package/runtime/infernet_liquid/bundle.py +63 -0
  27. package/runtime/infernet_liquid/canonical.py +75 -0
  28. package/runtime/infernet_liquid/data.py +121 -0
  29. package/runtime/infernet_liquid/errors.py +13 -0
  30. package/runtime/infernet_liquid/evaluate.py +145 -0
  31. package/runtime/infernet_liquid/export.py +136 -0
  32. package/runtime/infernet_liquid/features.py +105 -0
  33. package/runtime/infernet_liquid/models.py +63 -0
  34. package/runtime/infernet_liquid/schema.py +155 -0
  35. package/runtime/infernet_liquid/secretscan.py +64 -0
  36. package/runtime/infernet_liquid/server.py +251 -0
  37. package/runtime/infernet_liquid/sessions.py +279 -0
  38. package/runtime/infernet_liquid/synth.py +194 -0
  39. package/runtime/infernet_liquid/train.py +136 -0
  40. package/runtime/pyproject.toml +14 -0
  41. package/runtime/requirements-cuda.lock +461 -0
  42. package/runtime/requirements.lock +344 -0
  43. package/schemas/action-proposal.schema.json +40 -0
  44. package/schemas/common.schema.json +88 -0
  45. package/schemas/dataset-bundle.schema.json +15 -0
  46. package/schemas/dataset-manifest.schema.json +100 -0
  47. package/schemas/episode.schema.json +71 -0
  48. package/schemas/inference-session.schema.json +23 -0
  49. package/schemas/model-manifest.schema.json +103 -0
  50. package/schemas/predict-request.schema.json +20 -0
  51. package/schemas/train-job.schema.json +75 -0
  52. package/src/canonical.js +49 -0
  53. package/src/contract.js +62 -0
  54. package/src/dataset.js +133 -0
  55. package/src/index.js +13 -0
  56. package/src/runtime.js +206 -0
  57. package/src/secrets.js +79 -0
  58. package/src/validate.js +161 -0
@@ -0,0 +1,136 @@
1
+ """safetensors + ModelManifestV1 export and hash-verified loading."""
2
+
3
+ import hashlib
4
+ import os
5
+ import platform
6
+ from pathlib import Path
7
+
8
+ import torch
9
+ from safetensors.torch import load_file, save_file
10
+
11
+ from . import RUNTIME_NAME, SCHEMA_VERSION, __version__
12
+ from .canonical import file_sha256, manifest_hash
13
+ from .errors import LiquidError
14
+ from .features import ACTION_TYPES
15
+ from .models import TemporalPolicy
16
+ from .schema import validate
17
+ from .train import now_iso
18
+
19
+ RUNTIME_DIR = Path(__file__).resolve().parents[1]
20
+ WEIGHTS_FILE = "model.safetensors"
21
+ MANIFEST_FILE = "manifest.json"
22
+
23
+
24
+ def lock_hash():
25
+ lock = "requirements-cuda.lock" if os.environ.get("INFERNET_LIQUID_LOCK") == "cuda" else "requirements.lock"
26
+ for name in (lock, "pyproject.toml"):
27
+ p = RUNTIME_DIR / name
28
+ if p.exists():
29
+ return "sha256:" + hashlib.sha256(p.read_bytes()).hexdigest()
30
+ return "sha256:" + "0" * 64
31
+
32
+
33
+ def runtime_ref():
34
+ import ncps
35
+ from importlib.metadata import version
36
+
37
+ try:
38
+ ncps_v = version("ncps")
39
+ except Exception:
40
+ ncps_v = getattr(ncps, "__version__", "unknown")
41
+ return {
42
+ "name": RUNTIME_NAME,
43
+ "version": __version__,
44
+ "lock_hash": lock_hash(),
45
+ "python": platform.python_version(),
46
+ "torch": torch.__version__,
47
+ "ncps": ncps_v,
48
+ }
49
+
50
+
51
+ def save_model(out_dir, model, info, dataset_hash, eval_metrics, eval_split="test", eval_episodes=0, name=None, train_job_id=None, license="UNLICENSED-private", tasks=None):
52
+ out = Path(out_dir)
53
+ out.mkdir(parents=True, exist_ok=True)
54
+ wpath = out / WEIGHTS_FILE
55
+ state = {k: v.detach().cpu().contiguous().clone() for k, v in model.state_dict().items()}
56
+ save_file(state, str(wpath), metadata={"liquid_agent": f"{SCHEMA_VERSION};family={model.family}"}) # one key: safetensors orders a multi-key map randomly
57
+ weights_hash = file_sha256(wpath)
58
+ hp = info["hyperparameters"]
59
+ manifest = {
60
+ "schema_version": SCHEMA_VERSION,
61
+ "kind": "model_manifest",
62
+ "model_id": "lm_" + weights_hash[7:23],
63
+ "created_at": now_iso(),
64
+ "architecture": {
65
+ "family": model.family,
66
+ "hidden_units": model.hidden_units,
67
+ "feature_dim": model.feature_dim,
68
+ "backbone_units": model.backbone_units,
69
+ "params": model.param_count(),
70
+ "research": model.family == "ltc",
71
+ },
72
+ "weights": {"format": "safetensors", "file": WEIGHTS_FILE, "weights_hash": weights_hash, "size_bytes": os.path.getsize(wpath)},
73
+ "runtime": runtime_ref(),
74
+ "input_schema": {"encoding": "hashed-bag/v1", "feature_dim": model.feature_dim, "time_delta": "log1p_seconds", "tasks": sorted(tasks or [])},
75
+ "output_schema": {"action_types": list(ACTION_TYPES), "targets": ["pointer:visible-elements"]},
76
+ "training": {
77
+ "dataset_hash": dataset_hash,
78
+ "seed": int(hp["seed"]),
79
+ "epochs": int(info["epochs_run"]),
80
+ "learning_rate": float(hp["learning_rate"]),
81
+ "device": info["device"],
82
+ "started_at": info["started_at"],
83
+ "finished_at": info["finished_at"],
84
+ "train_seconds": info["train_seconds"],
85
+ },
86
+ "eval": {
87
+ "split": eval_split,
88
+ "episodes": int(eval_episodes),
89
+ "metrics": {k: v for k, v in eval_metrics.items() if isinstance(v, (int, float)) and not isinstance(v, bool)},
90
+ },
91
+ "license": license,
92
+ }
93
+ if name:
94
+ manifest["name"] = name
95
+ if train_job_id:
96
+ manifest["training"]["train_job_id"] = train_job_id
97
+ manifest["manifest_hash"] = manifest_hash(manifest)
98
+ errors = validate(manifest, "model_manifest")
99
+ if errors:
100
+ raise LiquidError("E_SCHEMA", f"exported manifest invalid: {errors[:3]}")
101
+ import json
102
+
103
+ (out / MANIFEST_FILE).write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", "utf-8")
104
+ return manifest
105
+
106
+
107
+ def read_manifest(model_dir):
108
+ import json
109
+
110
+ p = Path(model_dir) / MANIFEST_FILE
111
+ if not p.exists():
112
+ raise LiquidError("E_BAD_REQUEST", f"no {MANIFEST_FILE} in {model_dir}")
113
+ manifest = json.loads(p.read_text("utf-8"))
114
+ errors = validate(manifest, "model_manifest")
115
+ if errors:
116
+ raise LiquidError("E_SCHEMA", "model manifest failed validation: " + "; ".join(f"{a or '/'} {b}" for a, b in errors[:5]))
117
+ if manifest_hash(manifest) != manifest["manifest_hash"]:
118
+ raise LiquidError("E_HASH_MISMATCH", "manifest_hash does not match the manifest")
119
+ return manifest
120
+
121
+
122
+ def load_model(model_dir, expected_hash=None):
123
+ manifest = read_manifest(model_dir)
124
+ wpath = Path(model_dir) / manifest["weights"]["file"]
125
+ if not wpath.exists():
126
+ raise LiquidError("E_BAD_REQUEST", f"weights file missing: {wpath}")
127
+ actual = file_sha256(wpath)
128
+ if actual != manifest["weights"]["weights_hash"]:
129
+ raise LiquidError("E_HASH_MISMATCH", "weights do not match weights_hash in the manifest")
130
+ if expected_hash and expected_hash != actual:
131
+ raise LiquidError("E_MODEL_HASH", f"model hash {actual} is not the requested {expected_hash}")
132
+ a = manifest["architecture"]
133
+ model = TemporalPolicy(a["family"], a["feature_dim"], a["hidden_units"], a.get("backbone_units", 64))
134
+ model.load_state_dict(load_file(str(wpath)), strict=True)
135
+ model.eval()
136
+ return model, manifest
@@ -0,0 +1,105 @@
1
+ """hashed-bag/v1 featurisation of steps and of the visible UI elements.
2
+
3
+ Steps become a fixed-width vector from hashed tokens (task, app, url pattern,
4
+ focus, visible element words, previous action) plus log1p(seconds since the last
5
+ step). Elements become their own hashed vectors so the policy can *point* at a
6
+ visible element instead of choosing from a fixed name vocabulary; that is what
7
+ lets a model trained on "From Dana: Lunch?" click "From Sam: Invoice" later.
8
+ """
9
+
10
+ import hashlib
11
+ import math
12
+ import re
13
+
14
+ ACTION_TYPES = [
15
+ "click", "double_click", "type", "key", "scroll", "navigate", "select", "focus",
16
+ "wait", "submit", "send", "post", "publish", "delete", "done",
17
+ ]
18
+ EXTERNAL_WRITE = {"submit", "send", "post", "publish", "delete"}
19
+ TARGETED = {"click", "double_click", "type", "select", "focus", "submit", "send", "post", "publish", "delete"}
20
+
21
+ ELEMENT_DIM = 128
22
+ MAX_ELEMENTS = 64
23
+ WORD = re.compile(r"[a-z0-9]+")
24
+
25
+
26
+ def norm(s):
27
+ return " ".join(str(s or "").lower().split())[:80]
28
+
29
+
30
+ def element_key(el):
31
+ if not el:
32
+ return ""
33
+ return f"{norm(el.get('role'))}:{norm(el.get('name'))}"
34
+
35
+
36
+ def _bucket(token, dim):
37
+ d = hashlib.blake2b(token.encode("utf-8"), digest_size=8).digest()
38
+ n = int.from_bytes(d, "little")
39
+ return n % dim, (1.0 if (n >> 63) & 1 else -1.0)
40
+
41
+
42
+ def _add(vec, token, weight, dim):
43
+ i, s = _bucket(token, dim)
44
+ vec[i] += s * weight
45
+
46
+
47
+ def words(s):
48
+ return WORD.findall(norm(s))
49
+
50
+
51
+ def encode_step(task, observation, prev_action, dt_ms, feature_dim):
52
+ """Return a list[float] of length feature_dim. The last two slots are log1p(dt seconds) and a bias."""
53
+ dim = feature_dim - 2
54
+ v = [0.0] * feature_dim
55
+ obs = observation or {}
56
+ if task:
57
+ _add(v, f"task:{task}", 1.0, dim)
58
+ _add(v, f"app:{norm(obs.get('app'))}", 1.0, dim)
59
+ if obs.get("url_pattern"):
60
+ _add(v, f"url:{norm(obs['url_pattern'])}", 1.0, dim)
61
+ _add(v, f"loading:{bool(obs.get('loading'))}", 1.0, dim)
62
+ if obs.get("focus"):
63
+ _add(v, f"focus:{element_key(obs['focus'])}", 1.0, dim)
64
+ _add(v, f"focusrole:{norm(obs['focus'].get('role'))}", 1.0, dim)
65
+ els = (obs.get("elements") or [])[:MAX_ELEMENTS]
66
+ if els:
67
+ w = 1.0 / math.sqrt(len(els))
68
+ for el in els:
69
+ _add(v, f"elrole:{norm(el.get('role'))}", w, dim)
70
+ for word in words(el.get("name")):
71
+ _add(v, f"elword:{word}", w, dim)
72
+ if prev_action:
73
+ _add(v, f"pa:{prev_action.get('type')}", 1.0, dim)
74
+ if prev_action.get("target"):
75
+ _add(v, f"pt:{element_key(prev_action['target'])}", 0.5, dim)
76
+ _add(v, f"ptrole:{norm(prev_action['target'].get('role'))}", 1.0, dim)
77
+ else:
78
+ _add(v, "pa:<start>", 1.0, dim)
79
+ v[feature_dim - 2] = math.log1p(max(0, dt_ms or 0) / 1000.0)
80
+ v[feature_dim - 1] = 1.0
81
+ return v
82
+
83
+
84
+ def encode_element(el, index):
85
+ v = [0.0] * ELEMENT_DIM
86
+ dim = ELEMENT_DIM - 1
87
+ _add(v, f"role:{norm(el.get('role'))}", 1.0, dim)
88
+ ws = words(el.get("name"))
89
+ for w in ws:
90
+ _add(v, f"word:{w}", 1.0 / math.sqrt(len(ws)), dim)
91
+ _add(v, f"rw:{norm(el.get('role'))}:{w}", 0.5 / math.sqrt(len(ws)), dim)
92
+ _add(v, f"pos:{min(index, 8)}", 0.5, dim)
93
+ v[ELEMENT_DIM - 1] = 1.0
94
+ return v
95
+
96
+
97
+ def target_index(observation, action):
98
+ """Index of the action's target among the visible elements, -1 for no target, None if not visible."""
99
+ if not action or not action.get("target"):
100
+ return -1
101
+ key = element_key(action["target"])
102
+ for i, el in enumerate(((observation or {}).get("elements") or [])[:MAX_ELEMENTS]):
103
+ if element_key(el) == key:
104
+ return i
105
+ return None
@@ -0,0 +1,63 @@
1
+ """TemporalPolicy: a CfC, LTC or GRU core with an action-type head and a pointer over visible elements."""
2
+
3
+ import torch
4
+ from torch import nn
5
+
6
+ from .features import ACTION_TYPES, ELEMENT_DIM
7
+
8
+ PROJ_DIM = 64
9
+
10
+
11
+ class TemporalPolicy(nn.Module):
12
+ def __init__(self, family, feature_dim, hidden_units, backbone_units=64):
13
+ super().__init__()
14
+ self.family = family
15
+ self.feature_dim = feature_dim
16
+ self.hidden_units = hidden_units
17
+ self.backbone_units = backbone_units
18
+ self.proj = nn.Linear(feature_dim, PROJ_DIM)
19
+ if family == "cfc":
20
+ from ncps.torch import CfCCell
21
+
22
+ self.cell = CfCCell(PROJ_DIM, hidden_units, backbone_units=backbone_units, backbone_layers=1)
23
+ elif family == "ltc":
24
+ from ncps.torch import LTCCell
25
+ from ncps.wirings import FullyConnected
26
+
27
+ wiring = FullyConnected(hidden_units)
28
+ wiring.build(PROJ_DIM)
29
+ self.cell = LTCCell(wiring, PROJ_DIM, ode_unfolds=6, implicit_param_constraints=True)
30
+ elif family == "gru":
31
+ self.cell = nn.GRUCell(PROJ_DIM, hidden_units)
32
+ else:
33
+ raise ValueError(f"unknown family {family}")
34
+ self.type_head = nn.Linear(hidden_units, len(ACTION_TYPES))
35
+ self.elem_enc = nn.Linear(ELEMENT_DIM, hidden_units)
36
+ self.query = nn.Linear(hidden_units, hidden_units)
37
+
38
+ def init_state(self, batch):
39
+ return torch.zeros(batch, self.hidden_units)
40
+
41
+ def step(self, x, h, dt):
42
+ """x [B,F], h [B,H], dt [B,1] seconds → (core output [B,H], new state [B,H])."""
43
+ z = torch.tanh(self.proj(x))
44
+ if self.family == "gru":
45
+ h = self.cell(z, h)
46
+ return h, h
47
+ if self.family == "ltc":
48
+ # The LTC solver divides by elapsed time; a zero gap (first step) would be inf.
49
+ dt = dt.clamp(min=0.05)
50
+ out, h = self.cell(z, h, dt)
51
+ return out, h
52
+
53
+ def heads(self, out, elems, elem_mask):
54
+ """out [B,H]; elems [B,E,ELEMENT_DIM]; elem_mask [B,E] bool → (type logits [B,T], pointer logits [B,E])."""
55
+ type_logits = self.type_head(out)
56
+ keys = torch.tanh(self.elem_enc(elems))
57
+ q = self.query(out).unsqueeze(1)
58
+ ptr = (keys * q).sum(-1) / (self.hidden_units ** 0.5)
59
+ ptr = ptr.masked_fill(~elem_mask, -1e9)
60
+ return type_logits, ptr
61
+
62
+ def param_count(self):
63
+ return sum(p.numel() for p in self.parameters())
@@ -0,0 +1,155 @@
1
+ """Subset JSON Schema validator mirroring src/validate.js over the same schema files."""
2
+
3
+ import json
4
+ import re
5
+ from datetime import datetime
6
+ from pathlib import Path
7
+
8
+ SCHEMA_DIR = Path(__file__).resolve().parents[2] / "schemas"
9
+
10
+ SCHEMA_FILES = {
11
+ "episode": "episode.schema.json",
12
+ "dataset_manifest": "dataset-manifest.schema.json",
13
+ "dataset_bundle": "dataset-bundle.schema.json",
14
+ "train_job": "train-job.schema.json",
15
+ "model_manifest": "model-manifest.schema.json",
16
+ "inference_session": "inference-session.schema.json",
17
+ "predict_request": "predict-request.schema.json",
18
+ "action_proposal": "action-proposal.schema.json",
19
+ }
20
+
21
+ KNOWN = {
22
+ "$schema", "$id", "$defs", "$ref", "title", "description", "type", "const", "enum",
23
+ "properties", "required", "additionalProperties", "items", "minItems", "maxItems",
24
+ "uniqueItems", "minLength", "maxLength", "pattern", "minimum", "maximum",
25
+ "exclusiveMinimum", "anyOf", "format",
26
+ }
27
+
28
+ DATE_TIME = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?(Z|[+-]\d{2}:\d{2})$")
29
+
30
+ _registry = None
31
+
32
+
33
+ def _load():
34
+ global _registry
35
+ if _registry is None:
36
+ _registry = {p.name: json.loads(p.read_text("utf-8")) for p in sorted(SCHEMA_DIR.glob("*.schema.json"))}
37
+ return _registry
38
+
39
+
40
+ def _resolve(ref, base):
41
+ file, _, frag = ref.partition("#")
42
+ target = file or base
43
+ node = _load().get(target)
44
+ if node is None:
45
+ raise ValueError(f"schema $ref to unknown file: {ref}")
46
+ for part in [p for p in frag.split("/") if p]:
47
+ node = node[part]
48
+ return node, target
49
+
50
+
51
+ def _type_of(v):
52
+ if v is None:
53
+ return "null"
54
+ if isinstance(v, bool):
55
+ return "boolean"
56
+ if isinstance(v, int):
57
+ return "integer"
58
+ if isinstance(v, float):
59
+ return "integer" if v == int(v) else "number"
60
+ if isinstance(v, str):
61
+ return "string"
62
+ if isinstance(v, list):
63
+ return "array"
64
+ if isinstance(v, dict):
65
+ return "object"
66
+ return type(v).__name__
67
+
68
+
69
+ def _type_ok(want, v):
70
+ got = _type_of(v)
71
+ return got in ("number", "integer") if want == "number" else want == got
72
+
73
+
74
+ def _check(schema, value, at, file, errors):
75
+ unknown = set(schema) - KNOWN
76
+ if unknown:
77
+ raise ValueError(f"schema keyword not supported by the subset validator: {sorted(unknown)} ({file})")
78
+ if "$ref" in schema:
79
+ node, f = _resolve(schema["$ref"], file)
80
+ _check(node, value, at, f, errors)
81
+ if "anyOf" in schema:
82
+ for s in schema["anyOf"]:
83
+ e = []
84
+ _check(s, value, at, file, e)
85
+ if not e:
86
+ return
87
+ errors.append((at, "does not match any allowed shape"))
88
+ return
89
+ if "const" in schema and json.dumps(schema["const"]) != json.dumps(value):
90
+ errors.append((at, f"must equal {json.dumps(schema['const'])}"))
91
+ return
92
+ if "enum" in schema and not any(value == e and _type_of(value) == _type_of(e) for e in schema["enum"]):
93
+ errors.append((at, "must be one of " + ", ".join(map(str, schema["enum"]))))
94
+ return
95
+ if "type" in schema and not _type_ok(schema["type"], value):
96
+ errors.append((at, f"must be {schema['type']}"))
97
+ return
98
+ t = _type_of(value)
99
+ if t == "string":
100
+ n = len(value)
101
+ if "minLength" in schema and n < schema["minLength"]:
102
+ errors.append((at, f"shorter than {schema['minLength']}"))
103
+ if "maxLength" in schema and n > schema["maxLength"]:
104
+ errors.append((at, f"longer than {schema['maxLength']}"))
105
+ if "pattern" in schema and not re.search(schema["pattern"], value):
106
+ errors.append((at, f"does not match {schema['pattern']}"))
107
+ if schema.get("format") == "date-time":
108
+ ok = bool(DATE_TIME.match(value))
109
+ if ok:
110
+ try:
111
+ datetime.fromisoformat(value.replace("Z", "+00:00"))
112
+ except ValueError:
113
+ ok = False
114
+ if not ok:
115
+ errors.append((at, "not an RFC 3339 date-time"))
116
+ if t in ("number", "integer"):
117
+ if "minimum" in schema and value < schema["minimum"]:
118
+ errors.append((at, f"below {schema['minimum']}"))
119
+ if "maximum" in schema and value > schema["maximum"]:
120
+ errors.append((at, f"above {schema['maximum']}"))
121
+ if "exclusiveMinimum" in schema and value <= schema["exclusiveMinimum"]:
122
+ errors.append((at, f"must be > {schema['exclusiveMinimum']}"))
123
+ if t == "array":
124
+ if "minItems" in schema and len(value) < schema["minItems"]:
125
+ errors.append((at, f"fewer than {schema['minItems']} items"))
126
+ if "maxItems" in schema and len(value) > schema["maxItems"]:
127
+ errors.append((at, f"more than {schema['maxItems']} items"))
128
+ if schema.get("uniqueItems") and len({json.dumps(v, sort_keys=True) for v in value}) != len(value):
129
+ errors.append((at, "items must be unique"))
130
+ if "items" in schema:
131
+ for i, v in enumerate(value):
132
+ _check(schema["items"], v, f"{at}/{i}", file, errors)
133
+ if t == "object":
134
+ for req in schema.get("required", []):
135
+ if req not in value:
136
+ errors.append((f"{at}/{req}", "is required"))
137
+ props = schema.get("properties", {})
138
+ addl = schema.get("additionalProperties")
139
+ for k, v in value.items():
140
+ if k in props:
141
+ _check(props[k], v, f"{at}/{k}", file, errors)
142
+ elif addl is False:
143
+ errors.append((f"{at}/{k}", "is not allowed"))
144
+ elif isinstance(addl, dict):
145
+ _check(addl, v, f"{at}/{k}", file, errors)
146
+
147
+
148
+ def validate(value, kind=None):
149
+ kind = kind or (value.get("kind") if isinstance(value, dict) else None)
150
+ file = SCHEMA_FILES.get(kind)
151
+ if not file:
152
+ return [("/kind", f"unknown kind: {kind}")]
153
+ errors = []
154
+ _check(_load()[file], value, "", file, errors)
155
+ return errors
@@ -0,0 +1,64 @@
1
+ """Secret scan mirroring src/secrets.js; a second fence after redaction at capture."""
2
+
3
+ import re
4
+
5
+ SCANNER_VERSION = "1"
6
+
7
+ PATTERNS = [
8
+ ("private_key", re.compile(r"-----BEGIN [A-Z ]*PRIVATE KEY-----")),
9
+ ("jwt", re.compile(r"\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}")),
10
+ ("aws_access_key", re.compile(r"\b(AKIA|ASIA)[0-9A-Z]{16}\b")),
11
+ ("github_token", re.compile(r"\bgh[pousr]_[A-Za-z0-9]{30,}\b")),
12
+ ("slack_token", re.compile(r"\bxox[abprs]-[A-Za-z0-9-]{10,}")),
13
+ ("api_key", re.compile(r"\b(sk|pk|rk)[-_](live|test|proj|ant)?[-_]?[A-Za-z0-9]{20,}\b")),
14
+ ("bearer", re.compile(r"\bBearer\s+[A-Za-z0-9._~+/=-]{16,}", re.I)),
15
+ ("cookie", re.compile(r"\b(set-cookie|cookie)\s*:", re.I)),
16
+ ("password_assignment", re.compile(r"\b(pass(word)?|passwd|pwd|secret|api[_-]?key|token)\s*[:=]\s*\S{4,}", re.I)),
17
+ ]
18
+
19
+ STRUCTURAL = re.compile(r"(_hash|_ref|_id|_at|_ms|^seed$|^dt_ms$|^t_ms$)$")
20
+ CARDISH = re.compile(r"(?:\d[ -]?){13,19}")
21
+
22
+
23
+ def _luhn(digits):
24
+ total, alt = 0, False
25
+ for ch in reversed(digits):
26
+ d = ord(ch) - 48
27
+ if alt:
28
+ d *= 2
29
+ if d > 9:
30
+ d -= 9
31
+ total += d
32
+ alt = not alt
33
+ return total % 10 == 0
34
+
35
+
36
+ def scan_text(text):
37
+ hits = [name for name, rx in PATTERNS if rx.search(text)]
38
+ for m in CARDISH.finditer(text):
39
+ digits = re.sub(r"[ -]", "", m.group(0))
40
+ if 13 <= len(digits) <= 19 and _luhn(digits):
41
+ hits.append("card_number")
42
+ break
43
+ return hits
44
+
45
+
46
+ def scan_secrets(value):
47
+ out = []
48
+
49
+ def walk(v, path, key):
50
+ if isinstance(v, str):
51
+ if key and STRUCTURAL.search(key):
52
+ return
53
+ kinds = scan_text(v)
54
+ if kinds:
55
+ out.append({"path": path, "kinds": kinds})
56
+ elif isinstance(v, list):
57
+ for i, x in enumerate(v):
58
+ walk(x, f"{path}/{i}", key)
59
+ elif isinstance(v, dict):
60
+ for k, x in v.items():
61
+ walk(x, f"{path}/{k}", k)
62
+
63
+ walk(value, "", None)
64
+ return out