@infernetprotocol/liquid-agent 0.0.0-stage → 0.1.66
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +15 -0
- package/README.md +15 -2
- package/fixtures/contract.json +6 -0
- package/fixtures/invalid/episode-extra-field.json +275 -0
- package/fixtures/invalid/episode-no-capture.json +265 -0
- package/fixtures/invalid/episode-unknown-action.json +274 -0
- package/fixtures/invalid/model-manifest-bad-hash.json +92 -0
- package/fixtures/invalid/predict-negative-seq.json +48 -0
- package/fixtures/invalid/proposal-bad-abstain.json +31 -0
- package/fixtures/invalid/proposal-confidence-range.json +31 -0
- package/fixtures/invalid/session-bad-seq.json +17 -0
- package/fixtures/invalid/train-job-bad-family.json +44 -0
- package/fixtures/invalid/train-job-zero-lr.json +44 -0
- package/fixtures/valid/action-proposal.json +27 -0
- package/fixtures/valid/dataset-bundle.json +3747 -0
- package/fixtures/valid/dataset-manifest.json +126 -0
- package/fixtures/valid/episode.json +270 -0
- package/fixtures/valid/inference-session.json +13 -0
- package/fixtures/valid/model-manifest.json +88 -0
- package/fixtures/valid/predict-request.json +44 -0
- package/fixtures/valid/train-job.json +40 -0
- package/package.json +28 -3
- package/runtime/infernet_liquid/__init__.py +6 -0
- package/runtime/infernet_liquid/__main__.py +95 -0
- package/runtime/infernet_liquid/bench.py +125 -0
- package/runtime/infernet_liquid/bundle.py +63 -0
- package/runtime/infernet_liquid/canonical.py +75 -0
- package/runtime/infernet_liquid/data.py +121 -0
- package/runtime/infernet_liquid/errors.py +13 -0
- package/runtime/infernet_liquid/evaluate.py +145 -0
- package/runtime/infernet_liquid/export.py +136 -0
- package/runtime/infernet_liquid/features.py +105 -0
- package/runtime/infernet_liquid/models.py +63 -0
- package/runtime/infernet_liquid/schema.py +155 -0
- package/runtime/infernet_liquid/secretscan.py +64 -0
- package/runtime/infernet_liquid/server.py +251 -0
- package/runtime/infernet_liquid/sessions.py +279 -0
- package/runtime/infernet_liquid/synth.py +194 -0
- package/runtime/infernet_liquid/train.py +136 -0
- package/runtime/pyproject.toml +14 -0
- package/runtime/requirements-cuda.lock +461 -0
- package/runtime/requirements.lock +344 -0
- package/schemas/action-proposal.schema.json +40 -0
- package/schemas/common.schema.json +88 -0
- package/schemas/dataset-bundle.schema.json +15 -0
- package/schemas/dataset-manifest.schema.json +100 -0
- package/schemas/episode.schema.json +71 -0
- package/schemas/inference-session.schema.json +23 -0
- package/schemas/model-manifest.schema.json +103 -0
- package/schemas/predict-request.schema.json +20 -0
- package/schemas/train-job.schema.json +75 -0
- package/src/canonical.js +49 -0
- package/src/contract.js +62 -0
- package/src/dataset.js +133 -0
- package/src/index.js +13 -0
- package/src/runtime.js +206 -0
- package/src/secrets.js +79 -0
- package/src/validate.js +161 -0
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
"""safetensors + ModelManifestV1 export and hash-verified loading."""
|
|
2
|
+
|
|
3
|
+
import hashlib
|
|
4
|
+
import os
|
|
5
|
+
import platform
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
import torch
|
|
9
|
+
from safetensors.torch import load_file, save_file
|
|
10
|
+
|
|
11
|
+
from . import RUNTIME_NAME, SCHEMA_VERSION, __version__
|
|
12
|
+
from .canonical import file_sha256, manifest_hash
|
|
13
|
+
from .errors import LiquidError
|
|
14
|
+
from .features import ACTION_TYPES
|
|
15
|
+
from .models import TemporalPolicy
|
|
16
|
+
from .schema import validate
|
|
17
|
+
from .train import now_iso
|
|
18
|
+
|
|
19
|
+
RUNTIME_DIR = Path(__file__).resolve().parents[1]
|
|
20
|
+
WEIGHTS_FILE = "model.safetensors"
|
|
21
|
+
MANIFEST_FILE = "manifest.json"
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def lock_hash():
|
|
25
|
+
lock = "requirements-cuda.lock" if os.environ.get("INFERNET_LIQUID_LOCK") == "cuda" else "requirements.lock"
|
|
26
|
+
for name in (lock, "pyproject.toml"):
|
|
27
|
+
p = RUNTIME_DIR / name
|
|
28
|
+
if p.exists():
|
|
29
|
+
return "sha256:" + hashlib.sha256(p.read_bytes()).hexdigest()
|
|
30
|
+
return "sha256:" + "0" * 64
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def runtime_ref():
|
|
34
|
+
import ncps
|
|
35
|
+
from importlib.metadata import version
|
|
36
|
+
|
|
37
|
+
try:
|
|
38
|
+
ncps_v = version("ncps")
|
|
39
|
+
except Exception:
|
|
40
|
+
ncps_v = getattr(ncps, "__version__", "unknown")
|
|
41
|
+
return {
|
|
42
|
+
"name": RUNTIME_NAME,
|
|
43
|
+
"version": __version__,
|
|
44
|
+
"lock_hash": lock_hash(),
|
|
45
|
+
"python": platform.python_version(),
|
|
46
|
+
"torch": torch.__version__,
|
|
47
|
+
"ncps": ncps_v,
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def save_model(out_dir, model, info, dataset_hash, eval_metrics, eval_split="test", eval_episodes=0, name=None, train_job_id=None, license="UNLICENSED-private", tasks=None):
|
|
52
|
+
out = Path(out_dir)
|
|
53
|
+
out.mkdir(parents=True, exist_ok=True)
|
|
54
|
+
wpath = out / WEIGHTS_FILE
|
|
55
|
+
state = {k: v.detach().cpu().contiguous().clone() for k, v in model.state_dict().items()}
|
|
56
|
+
save_file(state, str(wpath), metadata={"liquid_agent": f"{SCHEMA_VERSION};family={model.family}"}) # one key: safetensors orders a multi-key map randomly
|
|
57
|
+
weights_hash = file_sha256(wpath)
|
|
58
|
+
hp = info["hyperparameters"]
|
|
59
|
+
manifest = {
|
|
60
|
+
"schema_version": SCHEMA_VERSION,
|
|
61
|
+
"kind": "model_manifest",
|
|
62
|
+
"model_id": "lm_" + weights_hash[7:23],
|
|
63
|
+
"created_at": now_iso(),
|
|
64
|
+
"architecture": {
|
|
65
|
+
"family": model.family,
|
|
66
|
+
"hidden_units": model.hidden_units,
|
|
67
|
+
"feature_dim": model.feature_dim,
|
|
68
|
+
"backbone_units": model.backbone_units,
|
|
69
|
+
"params": model.param_count(),
|
|
70
|
+
"research": model.family == "ltc",
|
|
71
|
+
},
|
|
72
|
+
"weights": {"format": "safetensors", "file": WEIGHTS_FILE, "weights_hash": weights_hash, "size_bytes": os.path.getsize(wpath)},
|
|
73
|
+
"runtime": runtime_ref(),
|
|
74
|
+
"input_schema": {"encoding": "hashed-bag/v1", "feature_dim": model.feature_dim, "time_delta": "log1p_seconds", "tasks": sorted(tasks or [])},
|
|
75
|
+
"output_schema": {"action_types": list(ACTION_TYPES), "targets": ["pointer:visible-elements"]},
|
|
76
|
+
"training": {
|
|
77
|
+
"dataset_hash": dataset_hash,
|
|
78
|
+
"seed": int(hp["seed"]),
|
|
79
|
+
"epochs": int(info["epochs_run"]),
|
|
80
|
+
"learning_rate": float(hp["learning_rate"]),
|
|
81
|
+
"device": info["device"],
|
|
82
|
+
"started_at": info["started_at"],
|
|
83
|
+
"finished_at": info["finished_at"],
|
|
84
|
+
"train_seconds": info["train_seconds"],
|
|
85
|
+
},
|
|
86
|
+
"eval": {
|
|
87
|
+
"split": eval_split,
|
|
88
|
+
"episodes": int(eval_episodes),
|
|
89
|
+
"metrics": {k: v for k, v in eval_metrics.items() if isinstance(v, (int, float)) and not isinstance(v, bool)},
|
|
90
|
+
},
|
|
91
|
+
"license": license,
|
|
92
|
+
}
|
|
93
|
+
if name:
|
|
94
|
+
manifest["name"] = name
|
|
95
|
+
if train_job_id:
|
|
96
|
+
manifest["training"]["train_job_id"] = train_job_id
|
|
97
|
+
manifest["manifest_hash"] = manifest_hash(manifest)
|
|
98
|
+
errors = validate(manifest, "model_manifest")
|
|
99
|
+
if errors:
|
|
100
|
+
raise LiquidError("E_SCHEMA", f"exported manifest invalid: {errors[:3]}")
|
|
101
|
+
import json
|
|
102
|
+
|
|
103
|
+
(out / MANIFEST_FILE).write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", "utf-8")
|
|
104
|
+
return manifest
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def read_manifest(model_dir):
|
|
108
|
+
import json
|
|
109
|
+
|
|
110
|
+
p = Path(model_dir) / MANIFEST_FILE
|
|
111
|
+
if not p.exists():
|
|
112
|
+
raise LiquidError("E_BAD_REQUEST", f"no {MANIFEST_FILE} in {model_dir}")
|
|
113
|
+
manifest = json.loads(p.read_text("utf-8"))
|
|
114
|
+
errors = validate(manifest, "model_manifest")
|
|
115
|
+
if errors:
|
|
116
|
+
raise LiquidError("E_SCHEMA", "model manifest failed validation: " + "; ".join(f"{a or '/'} {b}" for a, b in errors[:5]))
|
|
117
|
+
if manifest_hash(manifest) != manifest["manifest_hash"]:
|
|
118
|
+
raise LiquidError("E_HASH_MISMATCH", "manifest_hash does not match the manifest")
|
|
119
|
+
return manifest
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def load_model(model_dir, expected_hash=None):
|
|
123
|
+
manifest = read_manifest(model_dir)
|
|
124
|
+
wpath = Path(model_dir) / manifest["weights"]["file"]
|
|
125
|
+
if not wpath.exists():
|
|
126
|
+
raise LiquidError("E_BAD_REQUEST", f"weights file missing: {wpath}")
|
|
127
|
+
actual = file_sha256(wpath)
|
|
128
|
+
if actual != manifest["weights"]["weights_hash"]:
|
|
129
|
+
raise LiquidError("E_HASH_MISMATCH", "weights do not match weights_hash in the manifest")
|
|
130
|
+
if expected_hash and expected_hash != actual:
|
|
131
|
+
raise LiquidError("E_MODEL_HASH", f"model hash {actual} is not the requested {expected_hash}")
|
|
132
|
+
a = manifest["architecture"]
|
|
133
|
+
model = TemporalPolicy(a["family"], a["feature_dim"], a["hidden_units"], a.get("backbone_units", 64))
|
|
134
|
+
model.load_state_dict(load_file(str(wpath)), strict=True)
|
|
135
|
+
model.eval()
|
|
136
|
+
return model, manifest
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
"""hashed-bag/v1 featurisation of steps and of the visible UI elements.
|
|
2
|
+
|
|
3
|
+
Steps become a fixed-width vector from hashed tokens (task, app, url pattern,
|
|
4
|
+
focus, visible element words, previous action) plus log1p(seconds since the last
|
|
5
|
+
step). Elements become their own hashed vectors so the policy can *point* at a
|
|
6
|
+
visible element instead of choosing from a fixed name vocabulary; that is what
|
|
7
|
+
lets a model trained on "From Dana: Lunch?" click "From Sam: Invoice" later.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
import hashlib
|
|
11
|
+
import math
|
|
12
|
+
import re
|
|
13
|
+
|
|
14
|
+
ACTION_TYPES = [
|
|
15
|
+
"click", "double_click", "type", "key", "scroll", "navigate", "select", "focus",
|
|
16
|
+
"wait", "submit", "send", "post", "publish", "delete", "done",
|
|
17
|
+
]
|
|
18
|
+
EXTERNAL_WRITE = {"submit", "send", "post", "publish", "delete"}
|
|
19
|
+
TARGETED = {"click", "double_click", "type", "select", "focus", "submit", "send", "post", "publish", "delete"}
|
|
20
|
+
|
|
21
|
+
ELEMENT_DIM = 128
|
|
22
|
+
MAX_ELEMENTS = 64
|
|
23
|
+
WORD = re.compile(r"[a-z0-9]+")
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def norm(s):
|
|
27
|
+
return " ".join(str(s or "").lower().split())[:80]
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def element_key(el):
|
|
31
|
+
if not el:
|
|
32
|
+
return ""
|
|
33
|
+
return f"{norm(el.get('role'))}:{norm(el.get('name'))}"
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _bucket(token, dim):
|
|
37
|
+
d = hashlib.blake2b(token.encode("utf-8"), digest_size=8).digest()
|
|
38
|
+
n = int.from_bytes(d, "little")
|
|
39
|
+
return n % dim, (1.0 if (n >> 63) & 1 else -1.0)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _add(vec, token, weight, dim):
|
|
43
|
+
i, s = _bucket(token, dim)
|
|
44
|
+
vec[i] += s * weight
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def words(s):
|
|
48
|
+
return WORD.findall(norm(s))
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def encode_step(task, observation, prev_action, dt_ms, feature_dim):
|
|
52
|
+
"""Return a list[float] of length feature_dim. The last two slots are log1p(dt seconds) and a bias."""
|
|
53
|
+
dim = feature_dim - 2
|
|
54
|
+
v = [0.0] * feature_dim
|
|
55
|
+
obs = observation or {}
|
|
56
|
+
if task:
|
|
57
|
+
_add(v, f"task:{task}", 1.0, dim)
|
|
58
|
+
_add(v, f"app:{norm(obs.get('app'))}", 1.0, dim)
|
|
59
|
+
if obs.get("url_pattern"):
|
|
60
|
+
_add(v, f"url:{norm(obs['url_pattern'])}", 1.0, dim)
|
|
61
|
+
_add(v, f"loading:{bool(obs.get('loading'))}", 1.0, dim)
|
|
62
|
+
if obs.get("focus"):
|
|
63
|
+
_add(v, f"focus:{element_key(obs['focus'])}", 1.0, dim)
|
|
64
|
+
_add(v, f"focusrole:{norm(obs['focus'].get('role'))}", 1.0, dim)
|
|
65
|
+
els = (obs.get("elements") or [])[:MAX_ELEMENTS]
|
|
66
|
+
if els:
|
|
67
|
+
w = 1.0 / math.sqrt(len(els))
|
|
68
|
+
for el in els:
|
|
69
|
+
_add(v, f"elrole:{norm(el.get('role'))}", w, dim)
|
|
70
|
+
for word in words(el.get("name")):
|
|
71
|
+
_add(v, f"elword:{word}", w, dim)
|
|
72
|
+
if prev_action:
|
|
73
|
+
_add(v, f"pa:{prev_action.get('type')}", 1.0, dim)
|
|
74
|
+
if prev_action.get("target"):
|
|
75
|
+
_add(v, f"pt:{element_key(prev_action['target'])}", 0.5, dim)
|
|
76
|
+
_add(v, f"ptrole:{norm(prev_action['target'].get('role'))}", 1.0, dim)
|
|
77
|
+
else:
|
|
78
|
+
_add(v, "pa:<start>", 1.0, dim)
|
|
79
|
+
v[feature_dim - 2] = math.log1p(max(0, dt_ms or 0) / 1000.0)
|
|
80
|
+
v[feature_dim - 1] = 1.0
|
|
81
|
+
return v
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def encode_element(el, index):
|
|
85
|
+
v = [0.0] * ELEMENT_DIM
|
|
86
|
+
dim = ELEMENT_DIM - 1
|
|
87
|
+
_add(v, f"role:{norm(el.get('role'))}", 1.0, dim)
|
|
88
|
+
ws = words(el.get("name"))
|
|
89
|
+
for w in ws:
|
|
90
|
+
_add(v, f"word:{w}", 1.0 / math.sqrt(len(ws)), dim)
|
|
91
|
+
_add(v, f"rw:{norm(el.get('role'))}:{w}", 0.5 / math.sqrt(len(ws)), dim)
|
|
92
|
+
_add(v, f"pos:{min(index, 8)}", 0.5, dim)
|
|
93
|
+
v[ELEMENT_DIM - 1] = 1.0
|
|
94
|
+
return v
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def target_index(observation, action):
|
|
98
|
+
"""Index of the action's target among the visible elements, -1 for no target, None if not visible."""
|
|
99
|
+
if not action or not action.get("target"):
|
|
100
|
+
return -1
|
|
101
|
+
key = element_key(action["target"])
|
|
102
|
+
for i, el in enumerate(((observation or {}).get("elements") or [])[:MAX_ELEMENTS]):
|
|
103
|
+
if element_key(el) == key:
|
|
104
|
+
return i
|
|
105
|
+
return None
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
"""TemporalPolicy: a CfC, LTC or GRU core with an action-type head and a pointer over visible elements."""
|
|
2
|
+
|
|
3
|
+
import torch
|
|
4
|
+
from torch import nn
|
|
5
|
+
|
|
6
|
+
from .features import ACTION_TYPES, ELEMENT_DIM
|
|
7
|
+
|
|
8
|
+
PROJ_DIM = 64
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class TemporalPolicy(nn.Module):
|
|
12
|
+
def __init__(self, family, feature_dim, hidden_units, backbone_units=64):
|
|
13
|
+
super().__init__()
|
|
14
|
+
self.family = family
|
|
15
|
+
self.feature_dim = feature_dim
|
|
16
|
+
self.hidden_units = hidden_units
|
|
17
|
+
self.backbone_units = backbone_units
|
|
18
|
+
self.proj = nn.Linear(feature_dim, PROJ_DIM)
|
|
19
|
+
if family == "cfc":
|
|
20
|
+
from ncps.torch import CfCCell
|
|
21
|
+
|
|
22
|
+
self.cell = CfCCell(PROJ_DIM, hidden_units, backbone_units=backbone_units, backbone_layers=1)
|
|
23
|
+
elif family == "ltc":
|
|
24
|
+
from ncps.torch import LTCCell
|
|
25
|
+
from ncps.wirings import FullyConnected
|
|
26
|
+
|
|
27
|
+
wiring = FullyConnected(hidden_units)
|
|
28
|
+
wiring.build(PROJ_DIM)
|
|
29
|
+
self.cell = LTCCell(wiring, PROJ_DIM, ode_unfolds=6, implicit_param_constraints=True)
|
|
30
|
+
elif family == "gru":
|
|
31
|
+
self.cell = nn.GRUCell(PROJ_DIM, hidden_units)
|
|
32
|
+
else:
|
|
33
|
+
raise ValueError(f"unknown family {family}")
|
|
34
|
+
self.type_head = nn.Linear(hidden_units, len(ACTION_TYPES))
|
|
35
|
+
self.elem_enc = nn.Linear(ELEMENT_DIM, hidden_units)
|
|
36
|
+
self.query = nn.Linear(hidden_units, hidden_units)
|
|
37
|
+
|
|
38
|
+
def init_state(self, batch):
|
|
39
|
+
return torch.zeros(batch, self.hidden_units)
|
|
40
|
+
|
|
41
|
+
def step(self, x, h, dt):
|
|
42
|
+
"""x [B,F], h [B,H], dt [B,1] seconds → (core output [B,H], new state [B,H])."""
|
|
43
|
+
z = torch.tanh(self.proj(x))
|
|
44
|
+
if self.family == "gru":
|
|
45
|
+
h = self.cell(z, h)
|
|
46
|
+
return h, h
|
|
47
|
+
if self.family == "ltc":
|
|
48
|
+
# The LTC solver divides by elapsed time; a zero gap (first step) would be inf.
|
|
49
|
+
dt = dt.clamp(min=0.05)
|
|
50
|
+
out, h = self.cell(z, h, dt)
|
|
51
|
+
return out, h
|
|
52
|
+
|
|
53
|
+
def heads(self, out, elems, elem_mask):
|
|
54
|
+
"""out [B,H]; elems [B,E,ELEMENT_DIM]; elem_mask [B,E] bool → (type logits [B,T], pointer logits [B,E])."""
|
|
55
|
+
type_logits = self.type_head(out)
|
|
56
|
+
keys = torch.tanh(self.elem_enc(elems))
|
|
57
|
+
q = self.query(out).unsqueeze(1)
|
|
58
|
+
ptr = (keys * q).sum(-1) / (self.hidden_units ** 0.5)
|
|
59
|
+
ptr = ptr.masked_fill(~elem_mask, -1e9)
|
|
60
|
+
return type_logits, ptr
|
|
61
|
+
|
|
62
|
+
def param_count(self):
|
|
63
|
+
return sum(p.numel() for p in self.parameters())
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
"""Subset JSON Schema validator mirroring src/validate.js over the same schema files."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import re
|
|
5
|
+
from datetime import datetime
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
SCHEMA_DIR = Path(__file__).resolve().parents[2] / "schemas"
|
|
9
|
+
|
|
10
|
+
SCHEMA_FILES = {
|
|
11
|
+
"episode": "episode.schema.json",
|
|
12
|
+
"dataset_manifest": "dataset-manifest.schema.json",
|
|
13
|
+
"dataset_bundle": "dataset-bundle.schema.json",
|
|
14
|
+
"train_job": "train-job.schema.json",
|
|
15
|
+
"model_manifest": "model-manifest.schema.json",
|
|
16
|
+
"inference_session": "inference-session.schema.json",
|
|
17
|
+
"predict_request": "predict-request.schema.json",
|
|
18
|
+
"action_proposal": "action-proposal.schema.json",
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
KNOWN = {
|
|
22
|
+
"$schema", "$id", "$defs", "$ref", "title", "description", "type", "const", "enum",
|
|
23
|
+
"properties", "required", "additionalProperties", "items", "minItems", "maxItems",
|
|
24
|
+
"uniqueItems", "minLength", "maxLength", "pattern", "minimum", "maximum",
|
|
25
|
+
"exclusiveMinimum", "anyOf", "format",
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
DATE_TIME = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?(Z|[+-]\d{2}:\d{2})$")
|
|
29
|
+
|
|
30
|
+
_registry = None
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _load():
|
|
34
|
+
global _registry
|
|
35
|
+
if _registry is None:
|
|
36
|
+
_registry = {p.name: json.loads(p.read_text("utf-8")) for p in sorted(SCHEMA_DIR.glob("*.schema.json"))}
|
|
37
|
+
return _registry
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _resolve(ref, base):
|
|
41
|
+
file, _, frag = ref.partition("#")
|
|
42
|
+
target = file or base
|
|
43
|
+
node = _load().get(target)
|
|
44
|
+
if node is None:
|
|
45
|
+
raise ValueError(f"schema $ref to unknown file: {ref}")
|
|
46
|
+
for part in [p for p in frag.split("/") if p]:
|
|
47
|
+
node = node[part]
|
|
48
|
+
return node, target
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _type_of(v):
|
|
52
|
+
if v is None:
|
|
53
|
+
return "null"
|
|
54
|
+
if isinstance(v, bool):
|
|
55
|
+
return "boolean"
|
|
56
|
+
if isinstance(v, int):
|
|
57
|
+
return "integer"
|
|
58
|
+
if isinstance(v, float):
|
|
59
|
+
return "integer" if v == int(v) else "number"
|
|
60
|
+
if isinstance(v, str):
|
|
61
|
+
return "string"
|
|
62
|
+
if isinstance(v, list):
|
|
63
|
+
return "array"
|
|
64
|
+
if isinstance(v, dict):
|
|
65
|
+
return "object"
|
|
66
|
+
return type(v).__name__
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _type_ok(want, v):
|
|
70
|
+
got = _type_of(v)
|
|
71
|
+
return got in ("number", "integer") if want == "number" else want == got
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _check(schema, value, at, file, errors):
|
|
75
|
+
unknown = set(schema) - KNOWN
|
|
76
|
+
if unknown:
|
|
77
|
+
raise ValueError(f"schema keyword not supported by the subset validator: {sorted(unknown)} ({file})")
|
|
78
|
+
if "$ref" in schema:
|
|
79
|
+
node, f = _resolve(schema["$ref"], file)
|
|
80
|
+
_check(node, value, at, f, errors)
|
|
81
|
+
if "anyOf" in schema:
|
|
82
|
+
for s in schema["anyOf"]:
|
|
83
|
+
e = []
|
|
84
|
+
_check(s, value, at, file, e)
|
|
85
|
+
if not e:
|
|
86
|
+
return
|
|
87
|
+
errors.append((at, "does not match any allowed shape"))
|
|
88
|
+
return
|
|
89
|
+
if "const" in schema and json.dumps(schema["const"]) != json.dumps(value):
|
|
90
|
+
errors.append((at, f"must equal {json.dumps(schema['const'])}"))
|
|
91
|
+
return
|
|
92
|
+
if "enum" in schema and not any(value == e and _type_of(value) == _type_of(e) for e in schema["enum"]):
|
|
93
|
+
errors.append((at, "must be one of " + ", ".join(map(str, schema["enum"]))))
|
|
94
|
+
return
|
|
95
|
+
if "type" in schema and not _type_ok(schema["type"], value):
|
|
96
|
+
errors.append((at, f"must be {schema['type']}"))
|
|
97
|
+
return
|
|
98
|
+
t = _type_of(value)
|
|
99
|
+
if t == "string":
|
|
100
|
+
n = len(value)
|
|
101
|
+
if "minLength" in schema and n < schema["minLength"]:
|
|
102
|
+
errors.append((at, f"shorter than {schema['minLength']}"))
|
|
103
|
+
if "maxLength" in schema and n > schema["maxLength"]:
|
|
104
|
+
errors.append((at, f"longer than {schema['maxLength']}"))
|
|
105
|
+
if "pattern" in schema and not re.search(schema["pattern"], value):
|
|
106
|
+
errors.append((at, f"does not match {schema['pattern']}"))
|
|
107
|
+
if schema.get("format") == "date-time":
|
|
108
|
+
ok = bool(DATE_TIME.match(value))
|
|
109
|
+
if ok:
|
|
110
|
+
try:
|
|
111
|
+
datetime.fromisoformat(value.replace("Z", "+00:00"))
|
|
112
|
+
except ValueError:
|
|
113
|
+
ok = False
|
|
114
|
+
if not ok:
|
|
115
|
+
errors.append((at, "not an RFC 3339 date-time"))
|
|
116
|
+
if t in ("number", "integer"):
|
|
117
|
+
if "minimum" in schema and value < schema["minimum"]:
|
|
118
|
+
errors.append((at, f"below {schema['minimum']}"))
|
|
119
|
+
if "maximum" in schema and value > schema["maximum"]:
|
|
120
|
+
errors.append((at, f"above {schema['maximum']}"))
|
|
121
|
+
if "exclusiveMinimum" in schema and value <= schema["exclusiveMinimum"]:
|
|
122
|
+
errors.append((at, f"must be > {schema['exclusiveMinimum']}"))
|
|
123
|
+
if t == "array":
|
|
124
|
+
if "minItems" in schema and len(value) < schema["minItems"]:
|
|
125
|
+
errors.append((at, f"fewer than {schema['minItems']} items"))
|
|
126
|
+
if "maxItems" in schema and len(value) > schema["maxItems"]:
|
|
127
|
+
errors.append((at, f"more than {schema['maxItems']} items"))
|
|
128
|
+
if schema.get("uniqueItems") and len({json.dumps(v, sort_keys=True) for v in value}) != len(value):
|
|
129
|
+
errors.append((at, "items must be unique"))
|
|
130
|
+
if "items" in schema:
|
|
131
|
+
for i, v in enumerate(value):
|
|
132
|
+
_check(schema["items"], v, f"{at}/{i}", file, errors)
|
|
133
|
+
if t == "object":
|
|
134
|
+
for req in schema.get("required", []):
|
|
135
|
+
if req not in value:
|
|
136
|
+
errors.append((f"{at}/{req}", "is required"))
|
|
137
|
+
props = schema.get("properties", {})
|
|
138
|
+
addl = schema.get("additionalProperties")
|
|
139
|
+
for k, v in value.items():
|
|
140
|
+
if k in props:
|
|
141
|
+
_check(props[k], v, f"{at}/{k}", file, errors)
|
|
142
|
+
elif addl is False:
|
|
143
|
+
errors.append((f"{at}/{k}", "is not allowed"))
|
|
144
|
+
elif isinstance(addl, dict):
|
|
145
|
+
_check(addl, v, f"{at}/{k}", file, errors)
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def validate(value, kind=None):
|
|
149
|
+
kind = kind or (value.get("kind") if isinstance(value, dict) else None)
|
|
150
|
+
file = SCHEMA_FILES.get(kind)
|
|
151
|
+
if not file:
|
|
152
|
+
return [("/kind", f"unknown kind: {kind}")]
|
|
153
|
+
errors = []
|
|
154
|
+
_check(_load()[file], value, "", file, errors)
|
|
155
|
+
return errors
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
"""Secret scan mirroring src/secrets.js; a second fence after redaction at capture."""
|
|
2
|
+
|
|
3
|
+
import re
|
|
4
|
+
|
|
5
|
+
SCANNER_VERSION = "1"
|
|
6
|
+
|
|
7
|
+
PATTERNS = [
|
|
8
|
+
("private_key", re.compile(r"-----BEGIN [A-Z ]*PRIVATE KEY-----")),
|
|
9
|
+
("jwt", re.compile(r"\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}")),
|
|
10
|
+
("aws_access_key", re.compile(r"\b(AKIA|ASIA)[0-9A-Z]{16}\b")),
|
|
11
|
+
("github_token", re.compile(r"\bgh[pousr]_[A-Za-z0-9]{30,}\b")),
|
|
12
|
+
("slack_token", re.compile(r"\bxox[abprs]-[A-Za-z0-9-]{10,}")),
|
|
13
|
+
("api_key", re.compile(r"\b(sk|pk|rk)[-_](live|test|proj|ant)?[-_]?[A-Za-z0-9]{20,}\b")),
|
|
14
|
+
("bearer", re.compile(r"\bBearer\s+[A-Za-z0-9._~+/=-]{16,}", re.I)),
|
|
15
|
+
("cookie", re.compile(r"\b(set-cookie|cookie)\s*:", re.I)),
|
|
16
|
+
("password_assignment", re.compile(r"\b(pass(word)?|passwd|pwd|secret|api[_-]?key|token)\s*[:=]\s*\S{4,}", re.I)),
|
|
17
|
+
]
|
|
18
|
+
|
|
19
|
+
STRUCTURAL = re.compile(r"(_hash|_ref|_id|_at|_ms|^seed$|^dt_ms$|^t_ms$)$")
|
|
20
|
+
CARDISH = re.compile(r"(?:\d[ -]?){13,19}")
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _luhn(digits):
|
|
24
|
+
total, alt = 0, False
|
|
25
|
+
for ch in reversed(digits):
|
|
26
|
+
d = ord(ch) - 48
|
|
27
|
+
if alt:
|
|
28
|
+
d *= 2
|
|
29
|
+
if d > 9:
|
|
30
|
+
d -= 9
|
|
31
|
+
total += d
|
|
32
|
+
alt = not alt
|
|
33
|
+
return total % 10 == 0
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def scan_text(text):
|
|
37
|
+
hits = [name for name, rx in PATTERNS if rx.search(text)]
|
|
38
|
+
for m in CARDISH.finditer(text):
|
|
39
|
+
digits = re.sub(r"[ -]", "", m.group(0))
|
|
40
|
+
if 13 <= len(digits) <= 19 and _luhn(digits):
|
|
41
|
+
hits.append("card_number")
|
|
42
|
+
break
|
|
43
|
+
return hits
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def scan_secrets(value):
|
|
47
|
+
out = []
|
|
48
|
+
|
|
49
|
+
def walk(v, path, key):
|
|
50
|
+
if isinstance(v, str):
|
|
51
|
+
if key and STRUCTURAL.search(key):
|
|
52
|
+
return
|
|
53
|
+
kinds = scan_text(v)
|
|
54
|
+
if kinds:
|
|
55
|
+
out.append({"path": path, "kinds": kinds})
|
|
56
|
+
elif isinstance(v, list):
|
|
57
|
+
for i, x in enumerate(v):
|
|
58
|
+
walk(x, f"{path}/{i}", key)
|
|
59
|
+
elif isinstance(v, dict):
|
|
60
|
+
for k, x in v.items():
|
|
61
|
+
walk(x, f"{path}/{k}", k)
|
|
62
|
+
|
|
63
|
+
walk(value, "", None)
|
|
64
|
+
return out
|