brane 0.0.2 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (338) hide show
  1. package/AGENTS.md +362 -0
  2. package/CHANGELOG.md +33 -0
  3. package/CONTRIBUTING.md +91 -0
  4. package/DESIGN.md +402 -0
  5. package/Dockerfile +61 -0
  6. package/LICENSE +656 -17
  7. package/README.md +174 -2
  8. package/SECURITY.md +205 -0
  9. package/dist/absorb/brane-id.js +11 -0
  10. package/dist/absorb/chatgpt.js +161 -0
  11. package/dist/absorb/claude-code.js +504 -0
  12. package/dist/absorb/index.js +543 -0
  13. package/dist/absorb/organelle.js +345 -0
  14. package/dist/absorb/run.js +520 -0
  15. package/dist/absorb/sources.js +168 -0
  16. package/dist/absorb/types.js +3 -0
  17. package/dist/agent-env.js +45 -0
  18. package/dist/agents.js +1168 -0
  19. package/dist/claude-live-session.js +124 -0
  20. package/dist/cli.js +50 -0
  21. package/dist/codex-tool-log.js +144 -0
  22. package/dist/commands/absorb.js +291 -0
  23. package/dist/commands/attach.js +31 -0
  24. package/dist/commands/certify.js +62 -0
  25. package/dist/commands/config.js +116 -0
  26. package/dist/commands/cost.js +126 -0
  27. package/dist/commands/daemon.js +350 -0
  28. package/dist/commands/doctor.js +262 -0
  29. package/dist/commands/fold.js +95 -0
  30. package/dist/commands/list.js +89 -0
  31. package/dist/commands/mcp.js +84 -0
  32. package/dist/commands/meld.js +332 -0
  33. package/dist/commands/memory.js +215 -0
  34. package/dist/commands/revive.js +74 -0
  35. package/dist/commands/run.js +85 -0
  36. package/dist/commands/spawn.js +1094 -0
  37. package/dist/commands/token.js +153 -0
  38. package/dist/copilot-tool-log.js +145 -0
  39. package/dist/cost/pricing.js +116 -0
  40. package/dist/cost/reprice.js +136 -0
  41. package/dist/crush-tool-log.js +293 -0
  42. package/dist/daemon/agent-scope.js +116 -0
  43. package/dist/daemon/awaiting.js +54 -0
  44. package/dist/daemon/cert-manager.js +349 -0
  45. package/dist/daemon/client.js +262 -0
  46. package/dist/daemon/gated-routes.js +112 -0
  47. package/dist/daemon/keep-awake.js +103 -0
  48. package/dist/daemon/local-socket.js +145 -0
  49. package/dist/daemon/presence-gate.js +206 -0
  50. package/dist/daemon/server.js +6349 -0
  51. package/dist/daemon/signal-source.js +35 -0
  52. package/dist/daemon/tailscale-rebind.js +58 -0
  53. package/dist/daemon/token-ring.js +263 -0
  54. package/dist/daemon/token-routes.js +190 -0
  55. package/dist/db/events.js +152 -0
  56. package/dist/db/mentions.js +179 -0
  57. package/dist/db/schema.js +633 -0
  58. package/dist/db/tokens.js +583 -0
  59. package/dist/db/tool-uses.js +647 -0
  60. package/dist/diagram/index.js +85 -0
  61. package/dist/diagram/label.js +455 -0
  62. package/dist/diagram/layout.js +476 -0
  63. package/dist/diagram/links.js +50 -0
  64. package/dist/diagram/metrics.js +152 -0
  65. package/dist/diagram/parse.js +487 -0
  66. package/dist/diagram/svg.js +389 -0
  67. package/dist/diagram/types.js +9 -0
  68. package/dist/export.js +629 -0
  69. package/dist/files/archive-types.js +2 -0
  70. package/dist/files/archive.js +182 -0
  71. package/dist/files/browse.js +327 -0
  72. package/dist/files/bytes.js +162 -0
  73. package/dist/files/codecs.js +33 -0
  74. package/dist/files/find.js +212 -0
  75. package/dist/files/tar.js +223 -0
  76. package/dist/files/zip.js +209 -0
  77. package/dist/folds.js +72 -0
  78. package/dist/grammars/ebnf.js +92 -0
  79. package/dist/highlight.js +250 -0
  80. package/dist/ids.js +24 -0
  81. package/dist/markdown-diagnostics.js +233 -0
  82. package/dist/math.js +168 -0
  83. package/dist/mcp/server.js +886 -0
  84. package/dist/memory/embedder.js +145 -0
  85. package/dist/memory/embedding-worker.js +71 -0
  86. package/dist/memory/embeddings.js +91 -0
  87. package/dist/memory/index.js +232 -0
  88. package/dist/memory/storage.js +587 -0
  89. package/dist/memory/wiki.js +688 -0
  90. package/dist/memory/wordpiece.js +247 -0
  91. package/dist/mentions-scan.js +40 -0
  92. package/dist/presence.js +259 -0
  93. package/dist/session-affinity.js +185 -0
  94. package/dist/shared/api-request.js +124 -0
  95. package/dist/shared/block-glyphs.js +114 -0
  96. package/dist/shared/diagram-css.js +17 -0
  97. package/dist/shared/export-selection.js +101 -0
  98. package/dist/shared/find-in-text.js +110 -0
  99. package/dist/shared/glow-hues.js +225 -0
  100. package/dist/shared/hex-dump.js +62 -0
  101. package/dist/shared/hex-pages.js +52 -0
  102. package/dist/shared/hit-drum.js +79 -0
  103. package/dist/shared/known-nids.js +58 -0
  104. package/dist/shared/list-scroll-keys.js +56 -0
  105. package/dist/shared/markdown-lists.js +72 -0
  106. package/dist/shared/markdown-math.js +485 -0
  107. package/dist/shared/markdown-mentions.js +79 -0
  108. package/dist/shared/markdown-nids.js +41 -0
  109. package/dist/shared/markdown-protocols.js +53 -0
  110. package/dist/shared/markdown-refs.js +123 -0
  111. package/dist/shared/membrane-invasion.js +342 -0
  112. package/dist/shared/membrane-motion.js +83 -0
  113. package/dist/shared/membrane-palette.js +253 -0
  114. package/dist/shared/membrane-terrain.js +212 -0
  115. package/dist/shared/message-selection.js +123 -0
  116. package/dist/shared/mo-glyph.js +57 -0
  117. package/dist/shared/narrative-format.js +237 -0
  118. package/dist/shared/narrative-menu.js +82 -0
  119. package/dist/shared/narrative-sort.js +47 -0
  120. package/dist/shared/narrative-status.js +52 -0
  121. package/dist/shared/no-pan.js +85 -0
  122. package/dist/shared/pane-layout.js +50 -0
  123. package/dist/shared/payload-inline.js +83 -0
  124. package/dist/shared/speaker-color.js +61 -0
  125. package/dist/shared/token-rotation.js +14 -0
  126. package/dist/shared/view-url.js +197 -0
  127. package/dist/shared/wiki-links.js +70 -0
  128. package/dist/shared/wiki-window.js +4 -0
  129. package/dist/tailscale.js +55 -0
  130. package/dist/test/absorb-claude.test.js +892 -0
  131. package/dist/test/agent-scope.test.js +193 -0
  132. package/dist/test/agents.test.js +55 -0
  133. package/dist/test/api-request.test.js +122 -0
  134. package/dist/test/awaiting-glow.test.js +54 -0
  135. package/dist/test/awaiting.test.js +122 -0
  136. package/dist/test/bind-token.test.js +88 -0
  137. package/dist/test/block-glyphs.test.js +68 -0
  138. package/dist/test/cert-manager.test.js +262 -0
  139. package/dist/test/claude-live-session.test.js +110 -0
  140. package/dist/test/codex-tool-log.test.js +568 -0
  141. package/dist/test/config.test.js +71 -0
  142. package/dist/test/copilot-tool-log.test.js +93 -0
  143. package/dist/test/cors-preflight.test.js +49 -0
  144. package/dist/test/cost-linger.test.js +31 -0
  145. package/dist/test/cost-pricing.test.js +72 -0
  146. package/dist/test/cost-reprice.test.js +139 -0
  147. package/dist/test/cost-sessions.test.js +301 -0
  148. package/dist/test/crush-tool-log.test.js +289 -0
  149. package/dist/test/daemon-client.test.js +109 -0
  150. package/dist/test/daemon-socket.test.js +293 -0
  151. package/dist/test/diagram-arrows.test.js +205 -0
  152. package/dist/test/diagram-export.test.js +210 -0
  153. package/dist/test/diagram-labels.test.js +313 -0
  154. package/dist/test/diagram-parse.test.js +191 -0
  155. package/dist/test/diagram-render.test.js +175 -0
  156. package/dist/test/diagram-review.test.js +288 -0
  157. package/dist/test/diagram-self-loop-labels.test.js +303 -0
  158. package/dist/test/embedder.test.js +84 -0
  159. package/dist/test/events.test.js +123 -0
  160. package/dist/test/export-selection.test.js +102 -0
  161. package/dist/test/export.test.js +505 -0
  162. package/dist/test/files-browse.test.js +393 -0
  163. package/dist/test/files-find.test.js +272 -0
  164. package/dist/test/find-in-text.test.js +86 -0
  165. package/dist/test/first-paint-viewport.test.js +59 -0
  166. package/dist/test/fixtures/embedding-golden.js +23 -0
  167. package/dist/test/fixtures/wordpiece-golden.js +167 -0
  168. package/dist/test/glow-hues.test.js +202 -0
  169. package/dist/test/harness-cost.test.js +423 -0
  170. package/dist/test/helpers.js +21 -0
  171. package/dist/test/hex-pages.test.js +68 -0
  172. package/dist/test/highlight.test.js +69 -0
  173. package/dist/test/hit-drum.test.js +71 -0
  174. package/dist/test/hook-config.test.js +262 -0
  175. package/dist/test/json-gzip.test.js +70 -0
  176. package/dist/test/key-routing.test.js +58 -0
  177. package/dist/test/known-nids.test.js +106 -0
  178. package/dist/test/link-preview.test.js +67 -0
  179. package/dist/test/list-scroll-keys.test.js +66 -0
  180. package/dist/test/listen-own-words.test.js +147 -0
  181. package/dist/test/listen-timeout.test.js +33 -0
  182. package/dist/test/markdown-diagnostics.test.js +121 -0
  183. package/dist/test/markdown-lists.test.js +182 -0
  184. package/dist/test/markdown-math.test.js +256 -0
  185. package/dist/test/markdown-nids.test.js +146 -0
  186. package/dist/test/markdown-protocols.test.js +139 -0
  187. package/dist/test/mcp-bridge-cancel.test.js +98 -0
  188. package/dist/test/mcp-bridge-reconnect.test.js +121 -0
  189. package/dist/test/mcp-narrative.test.js +193 -0
  190. package/dist/test/mcp.test.js +548 -0
  191. package/dist/test/meld.test.js +85 -0
  192. package/dist/test/membrane-invasion.test.js +482 -0
  193. package/dist/test/membrane-motion.test.js +101 -0
  194. package/dist/test/membrane-palette.test.js +166 -0
  195. package/dist/test/membrane-terrain.test.js +170 -0
  196. package/dist/test/memory-log.test.js +202 -0
  197. package/dist/test/mentions-e2e.test.js +261 -0
  198. package/dist/test/mentions-ledger.test.js +265 -0
  199. package/dist/test/mentions-listen.test.js +265 -0
  200. package/dist/test/mentions.test.js +238 -0
  201. package/dist/test/message-selection.test.js +114 -0
  202. package/dist/test/mo-glyph.test.js +22 -0
  203. package/dist/test/model-resolution.test.js +166 -0
  204. package/dist/test/narrate-validate.test.js +77 -0
  205. package/dist/test/narrate-wait.test.js +160 -0
  206. package/dist/test/narrative-format.test.js +219 -0
  207. package/dist/test/narrative-menu.test.js +137 -0
  208. package/dist/test/narrative-sort.test.js +75 -0
  209. package/dist/test/no-pan.test.js +60 -0
  210. package/dist/test/pane-layout.test.js +153 -0
  211. package/dist/test/paths.test.js +37 -0
  212. package/dist/test/payload-inline.test.js +91 -0
  213. package/dist/test/precache-fonts.test.js +72 -0
  214. package/dist/test/presence-gate.test.js +440 -0
  215. package/dist/test/presence.test.js +160 -0
  216. package/dist/test/read-events.test.js +220 -0
  217. package/dist/test/remote-meld.test.js +115 -0
  218. package/dist/test/revive-cli.test.js +93 -0
  219. package/dist/test/search-floor.test.js +206 -0
  220. package/dist/test/search-fusion.test.js +168 -0
  221. package/dist/test/search.test.js +205 -0
  222. package/dist/test/security.test.js +48 -0
  223. package/dist/test/selection-wiring.test.js +61 -0
  224. package/dist/test/session-affinity.test.js +295 -0
  225. package/dist/test/signal-source.test.js +32 -0
  226. package/dist/test/socket-account.test.js +268 -0
  227. package/dist/test/spawn-toml.test.js +72 -0
  228. package/dist/test/speaker-color.test.js +90 -0
  229. package/dist/test/tailscale-rebind.test.js +79 -0
  230. package/dist/test/terminal-cursor.test.js +61 -0
  231. package/dist/test/terminal-resync.test.js +23 -0
  232. package/dist/test/thin-cli.test.js +382 -0
  233. package/dist/test/token-bootstrap.test.js +272 -0
  234. package/dist/test/token-devices.test.js +587 -0
  235. package/dist/test/token-lapse.test.js +94 -0
  236. package/dist/test/token-ring.test.js +188 -0
  237. package/dist/test/token-rotate-cli.test.js +58 -0
  238. package/dist/test/token-routes.test.js +95 -0
  239. package/dist/test/tool-uses.test.js +1006 -0
  240. package/dist/test/usage-ledger.test.js +126 -0
  241. package/dist/test/view-url.test.js +581 -0
  242. package/dist/test/wiki-links.test.js +57 -0
  243. package/dist/test/wiki-sources.test.js +85 -0
  244. package/dist/test/wiki-window.test.js +31 -0
  245. package/dist/test/wiki.test.js +399 -0
  246. package/dist/test/wordpiece.test.js +68 -0
  247. package/dist/tls.js +148 -0
  248. package/dist/tmux.js +202 -0
  249. package/dist/user-agent.js +46 -0
  250. package/docker-entrypoint.sh +141 -0
  251. package/docs/README.md +3 -0
  252. package/docs/absorb.md +231 -0
  253. package/docs/cost-ledger.md +125 -0
  254. package/docs/daemon-api.md +189 -0
  255. package/docs/diagrams.md +163 -0
  256. package/docs/doctor.md +36 -0
  257. package/docs/export-scope.md +69 -0
  258. package/docs/files-view.md +133 -0
  259. package/docs/link-previews.md +38 -0
  260. package/docs/math.md +32 -0
  261. package/docs/mcp.md +69 -0
  262. package/docs/memory-search.md +75 -0
  263. package/docs/mentions.md +101 -0
  264. package/docs/multi-brane.md +14 -0
  265. package/docs/passkeys.md +165 -0
  266. package/docs/presence.md +202 -0
  267. package/docs/releasing.md +53 -0
  268. package/docs/rendering-warnings.md +22 -0
  269. package/docs/request-layer.md +72 -0
  270. package/docs/selection-store.md +50 -0
  271. package/docs/session-affinity.md +61 -0
  272. package/docs/terminal-mirroring.md +211 -0
  273. package/docs/token-rotation.md +413 -0
  274. package/docs/view-urls.md +114 -0
  275. package/docs/web-app.md +23 -0
  276. package/docs/wiki.md +43 -0
  277. package/expo/dist/_expo/static/js/web/entry-e56efc1291dca7258ea00b8108b033a4.js +1097 -0
  278. package/expo/dist/apple-touch-icon.png +0 -0
  279. package/expo/dist/favicon.svg +3 -0
  280. package/expo/dist/index.html +177 -0
  281. package/expo/dist/katex/fonts/KaTeX_AMS-Regular.woff2 +0 -0
  282. package/expo/dist/katex/fonts/KaTeX_Caligraphic-Bold.woff2 +0 -0
  283. package/expo/dist/katex/fonts/KaTeX_Caligraphic-Regular.woff2 +0 -0
  284. package/expo/dist/katex/fonts/KaTeX_Fraktur-Bold.woff2 +0 -0
  285. package/expo/dist/katex/fonts/KaTeX_Fraktur-Regular.woff2 +0 -0
  286. package/expo/dist/katex/fonts/KaTeX_Main-Bold.woff2 +0 -0
  287. package/expo/dist/katex/fonts/KaTeX_Main-BoldItalic.woff2 +0 -0
  288. package/expo/dist/katex/fonts/KaTeX_Main-Italic.woff2 +0 -0
  289. package/expo/dist/katex/fonts/KaTeX_Main-Regular.woff2 +0 -0
  290. package/expo/dist/katex/fonts/KaTeX_Math-BoldItalic.woff2 +0 -0
  291. package/expo/dist/katex/fonts/KaTeX_Math-Italic.woff2 +0 -0
  292. package/expo/dist/katex/fonts/KaTeX_SansSerif-Bold.woff2 +0 -0
  293. package/expo/dist/katex/fonts/KaTeX_SansSerif-Italic.woff2 +0 -0
  294. package/expo/dist/katex/fonts/KaTeX_SansSerif-Regular.woff2 +0 -0
  295. package/expo/dist/katex/fonts/KaTeX_Script-Regular.woff2 +0 -0
  296. package/expo/dist/katex/fonts/KaTeX_Size1-Regular.woff2 +0 -0
  297. package/expo/dist/katex/fonts/KaTeX_Size2-Regular.woff2 +0 -0
  298. package/expo/dist/katex/fonts/KaTeX_Size3-Regular.woff2 +0 -0
  299. package/expo/dist/katex/fonts/KaTeX_Size4-Regular.woff2 +0 -0
  300. package/expo/dist/katex/fonts/KaTeX_Typewriter-Regular.woff2 +0 -0
  301. package/expo/dist/katex/katex.min.css +1 -0
  302. package/expo/dist/manifest.json +16 -0
  303. package/expo/dist/metadata.json +1 -0
  304. package/expo/dist/og/membrane.v1.png +0 -0
  305. package/expo/dist/pdf.min.mjs +28 -0
  306. package/expo/dist/pdf.worker.min.mjs +28 -0
  307. package/expo/dist/sw.js +154 -0
  308. package/package.json +81 -23
  309. package/protocols/LIVE_loop.md +407 -0
  310. package/protocols/bebrief.md +41 -0
  311. package/protocols/digest.md +231 -0
  312. package/protocols/fly.md +266 -0
  313. package/protocols/idle.md +81 -0
  314. package/protocols/keeper.md +829 -0
  315. package/protocols/markdown.md +80 -0
  316. package/protocols/memory.md +419 -0
  317. package/protocols/mentions.md +93 -0
  318. package/protocols/payloads.md +53 -0
  319. package/protocols/prelude.md +91 -0
  320. package/protocols/ramify.md +105 -0
  321. package/protocols/review.md +356 -0
  322. package/protocols/rrr.md +232 -0
  323. package/protocols/scribe.md +429 -0
  324. package/protocols/summarize.md +203 -0
  325. package/protocols/twin.md +174 -0
  326. package/protocols/wiki.md +438 -0
  327. package/protocols/worker.md +38 -0
  328. package/wiki/shared/append-only-event-log.md +148 -0
  329. package/wiki/shared/index.md +48 -0
  330. package/wiki/shared/owning-your-narratives.md +161 -0
  331. package/wiki/shared/premature-ontologies.md +435 -0
  332. package/wiki/shared/sedimentary-superintelligence.md +128 -0
  333. package/wiki/shared/sequestering-the-narrative.md +143 -0
  334. package/wiki/shared/why-wikis-work.md +216 -0
  335. package/.npmignore +0 -2
  336. package/.travis.yml +0 -5
  337. package/index.js +0 -8
  338. package/index.js~ +0 -12
@@ -0,0 +1,247 @@
1
+ /**
2
+ * A BERT WordPiece tokenizer, read straight from a HuggingFace `tokenizer.json`.
3
+ *
4
+ * This exists so brane can embed text without `@huggingface/transformers`,
5
+ * whose omnibus package drags in `onnxruntime-web` (91 MB) and `sharp` (whose
6
+ * libvips advisories are the only findings in a root `npm audit`) purely to
7
+ * support modalities and runtimes brane never uses. What brane actually needs
8
+ * from that package is this file plus one ONNX session.
9
+ *
10
+ * The implementation is a deliberate port of transformers.js's own
11
+ * `BertNormalizer`, `BertPreTokenizer` and `WordPieceTokenizer`, not an
12
+ * independent reading of the BERT paper. Token ids must match the previous
13
+ * pipeline *exactly*: `memory_vec` already holds vectors embedded the old way,
14
+ * and a tokenizer that disagrees on even a rare word doesn't fail — it quietly
15
+ * shifts query vectors away from the stored ones and degrades search in a way
16
+ * no error surfaces. `src/test/wordpiece.test.ts` pins the agreement.
17
+ */
18
+ import { readFileSync } from "node:fs";
19
+ /** Matches transformers.js's PUNCTUATION_REGEX (`src/tokenizers.js`): Unicode
20
+ * punctuation plus the ASCII symbol ranges BERT also treats as punctuation. */
21
+ const PUNCTUATION = "\\p{P}\\u0021-\\u002F\\u003A-\\u0040\\u005B-\\u0060\\u007B-\\u007E";
22
+ /** CJK ranges that BERT surrounds with whitespace so each char is its own token. */
23
+ function isChineseChar(cp) {
24
+ return ((cp >= 0x4e00 && cp <= 0x9fff) ||
25
+ (cp >= 0x3400 && cp <= 0x4dbf) ||
26
+ (cp >= 0x20000 && cp <= 0x2a6df) ||
27
+ (cp >= 0x2a700 && cp <= 0x2b73f) ||
28
+ (cp >= 0x2b740 && cp <= 0x2b81f) ||
29
+ (cp >= 0x2b820 && cp <= 0x2ceaf) ||
30
+ (cp >= 0xf900 && cp <= 0xfaff) ||
31
+ (cp >= 0x2f800 && cp <= 0x2fa1f));
32
+ }
33
+ /** Control characters, excluding the three whitespace ones BERT keeps. */
34
+ const CONTROL_RE = /^\p{Cc}|\p{Cf}|\p{Co}|\p{Cs}$/u;
35
+ function isControl(char) {
36
+ if (char === "\t" || char === "\n" || char === "\r")
37
+ return false;
38
+ return CONTROL_RE.test(char);
39
+ }
40
+ export class WordPieceTokenizer {
41
+ vocab;
42
+ unkToken;
43
+ prefix;
44
+ maxCharsPerWord;
45
+ norm;
46
+ preTokenizePattern;
47
+ addedPattern;
48
+ addedIds;
49
+ clsId;
50
+ sepId;
51
+ padId;
52
+ unkId;
53
+ /** Longest sequence the model's position embeddings cover, `[CLS]`/`[SEP]` included. */
54
+ modelMaxLength;
55
+ constructor(json, modelMaxLength = 512) {
56
+ if (json.model.type !== "WordPiece") {
57
+ throw new Error(`Unsupported tokenizer model: ${json.model.type}`);
58
+ }
59
+ if (json.normalizer?.type !== "BertNormalizer") {
60
+ throw new Error(`Unsupported normalizer: ${json.normalizer?.type}`);
61
+ }
62
+ if (json.pre_tokenizer?.type !== "BertPreTokenizer") {
63
+ throw new Error(`Unsupported pre-tokenizer: ${json.pre_tokenizer?.type}`);
64
+ }
65
+ this.vocab = new Map(Object.entries(json.model.vocab));
66
+ this.unkToken = json.model.unk_token;
67
+ const unkId = this.vocab.get(this.unkToken);
68
+ if (unkId === undefined)
69
+ throw new Error("unk_token missing from vocab");
70
+ this.unkId = unkId;
71
+ this.prefix = json.model.continuing_subword_prefix;
72
+ this.maxCharsPerWord = json.model.max_input_chars_per_word ?? 100;
73
+ this.norm = json.normalizer;
74
+ this.modelMaxLength = modelMaxLength;
75
+ // Equivalent to "strip whitespace, split on punctuation" — the same pattern
76
+ // transformers.js builds to match the Rust implementation.
77
+ this.preTokenizePattern = new RegExp(`[^\\s${PUNCTUATION}]+|[${PUNCTUATION}]`, "gu");
78
+ // Added tokens are matched against the *raw* text before normalization, so
79
+ // a literal "[CLS]" appearing in a narration tokenizes as the special token
80
+ // — matching the reference implementation rather than being lowercased into
81
+ // pieces. Longest-first so "[MASK]" can't be shadowed by a shorter entry.
82
+ const added = (json.added_tokens ?? []).slice().sort((a, b) => b.content.length - a.content.length);
83
+ this.addedIds = new Map(added.map((t) => [t.content, t.id]));
84
+ this.addedPattern = added.length
85
+ ? new RegExp(added.map((t) => escapeRegExp(t.content)).join("|"), "g")
86
+ : null;
87
+ this.clsId = this.requireId("[CLS]");
88
+ this.sepId = this.requireId("[SEP]");
89
+ this.padId = this.requireId("[PAD]");
90
+ }
91
+ /** Load from a HuggingFace `tokenizer.json` on disk. */
92
+ static fromFile(path, modelMaxLength = 512) {
93
+ return new WordPieceTokenizer(JSON.parse(readFileSync(path, "utf8")), modelMaxLength);
94
+ }
95
+ requireId(token) {
96
+ const id = this.addedIds.get(token) ?? this.vocab.get(token);
97
+ if (id === undefined)
98
+ throw new Error(`Required token missing: ${token}`);
99
+ return id;
100
+ }
101
+ /** BertNormalizer: clean → space out CJK → lowercase → strip accents. */
102
+ normalize(text) {
103
+ if (this.norm.clean_text) {
104
+ let out = "";
105
+ for (const char of text) {
106
+ const cp = char.codePointAt(0);
107
+ if (cp === 0 || cp === 0xfffd || isControl(char))
108
+ continue;
109
+ out += /^\s$/.test(char) ? " " : char;
110
+ }
111
+ text = out;
112
+ }
113
+ if (this.norm.handle_chinese_chars) {
114
+ let out = "";
115
+ for (let i = 0; i < text.length; i++) {
116
+ const char = text[i];
117
+ out += isChineseChar(char.charCodeAt(0)) ? ` ${char} ` : char;
118
+ }
119
+ text = out;
120
+ }
121
+ if (this.norm.lowercase) {
122
+ text = text.toLowerCase();
123
+ // `strip_accents: null` means "follow lowercase" — only an explicit
124
+ // `false` disables it. bge-small-en-v1.5 ships null, so accents go.
125
+ if (this.norm.strip_accents !== false)
126
+ text = stripAccents(text);
127
+ }
128
+ else if (this.norm.strip_accents) {
129
+ text = stripAccents(text);
130
+ }
131
+ return text;
132
+ }
133
+ /** Greedy longest-match-first WordPiece over one pre-token. */
134
+ wordPiece(token) {
135
+ const chars = [...token];
136
+ if (chars.length > this.maxCharsPerWord)
137
+ return [this.unkToken];
138
+ const sub = [];
139
+ let start = 0;
140
+ while (start < chars.length) {
141
+ let end = chars.length;
142
+ let match = null;
143
+ while (start < end) {
144
+ const candidate = (start > 0 ? this.prefix : "") + chars.slice(start, end).join("");
145
+ if (this.vocab.has(candidate)) {
146
+ match = candidate;
147
+ break;
148
+ }
149
+ --end;
150
+ }
151
+ // A single unmatched piece makes the *whole* word unknown — not just the
152
+ // offending span. That is BERT's rule and dropping it would silently
153
+ // change ids for any word containing an unseen character.
154
+ if (match === null)
155
+ return [this.unkToken];
156
+ sub.push(match);
157
+ start = end;
158
+ }
159
+ return sub;
160
+ }
161
+ /** Text → vocabulary ids, without `[CLS]`/`[SEP]` and without truncation. */
162
+ tokenize(text) {
163
+ const ids = [];
164
+ const segments = this.addedPattern
165
+ ? splitKeepingDelimiters(text, this.addedPattern)
166
+ : [{ text, isAdded: false }];
167
+ for (const seg of segments) {
168
+ if (seg.isAdded) {
169
+ ids.push(this.addedIds.get(seg.text));
170
+ continue;
171
+ }
172
+ const normalized = this.normalize(seg.text);
173
+ const preTokens = normalized.trim().match(this.preTokenizePattern) ?? [];
174
+ for (const pre of preTokens) {
175
+ for (const piece of this.wordPiece(pre)) {
176
+ ids.push(this.vocab.get(piece) ?? this.unkId);
177
+ }
178
+ }
179
+ }
180
+ return ids;
181
+ }
182
+ /**
183
+ * Encode one text the way the feature-extraction pipeline does: wrap in
184
+ * `[CLS]`/`[SEP]`, then truncate to the model's position-embedding limit.
185
+ *
186
+ * Order matters, and not in the way it first looks. The obvious reading —
187
+ * reserve two slots so a long text still ends in `[SEP]` — is *wrong* here,
188
+ * because HuggingFace truncates the already-post-processed sequence: an
189
+ * over-long text comes back 512 ids ending on an ordinary word, with the
190
+ * closing `[SEP]` cut off. Reserving room instead put `[SEP]` at index 511
191
+ * and disagreed with the reference on every text over the limit — 13% of
192
+ * this corpus, all of them long ones. Matching the reference is the whole
193
+ * point of this file, so the surprising behaviour is the correct one.
194
+ */
195
+ encode(text) {
196
+ const ids = this.tokenize(text);
197
+ let input_ids = [this.clsId, ...ids, this.sepId];
198
+ if (input_ids.length > this.modelMaxLength) {
199
+ input_ids = input_ids.slice(0, this.modelMaxLength);
200
+ }
201
+ return {
202
+ input_ids,
203
+ attention_mask: new Array(input_ids.length).fill(1),
204
+ token_type_ids: new Array(input_ids.length).fill(0),
205
+ };
206
+ }
207
+ /**
208
+ * Encode a batch, right-padding to the longest member. Padding positions get
209
+ * `attention_mask: 0` so mean pooling ignores them — a batch must produce the
210
+ * same vector for a text as encoding it alone would.
211
+ */
212
+ encodeBatch(texts) {
213
+ const encodings = texts.map((t) => this.encode(t));
214
+ const width = encodings.reduce((m, e) => Math.max(m, e.input_ids.length), 0);
215
+ const pad = (row, fill) => [
216
+ ...row,
217
+ ...new Array(width - row.length).fill(fill),
218
+ ];
219
+ return {
220
+ input_ids: encodings.map((e) => pad(e.input_ids, this.padId)),
221
+ attention_mask: encodings.map((e) => pad(e.attention_mask, 0)),
222
+ token_type_ids: encodings.map((e) => pad(e.token_type_ids, 0)),
223
+ };
224
+ }
225
+ }
226
+ /** "Mark, Nonspacing" removal after NFD — BERT's accent stripping. */
227
+ function stripAccents(text) {
228
+ return text.normalize("NFD").replace(/\p{Mn}/gu, "");
229
+ }
230
+ function escapeRegExp(s) {
231
+ return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
232
+ }
233
+ function splitKeepingDelimiters(text, pattern) {
234
+ const out = [];
235
+ let last = 0;
236
+ pattern.lastIndex = 0;
237
+ for (let m = pattern.exec(text); m !== null; m = pattern.exec(text)) {
238
+ if (m.index > last)
239
+ out.push({ text: text.slice(last, m.index), isAdded: false });
240
+ out.push({ text: m[0], isAdded: true });
241
+ last = m.index + m[0].length;
242
+ }
243
+ if (last < text.length)
244
+ out.push({ text: text.slice(last), isAdded: false });
245
+ return out;
246
+ }
247
+ //# sourceMappingURL=wordpiece.js.map
@@ -0,0 +1,40 @@
1
+ /** Extracting mentions from markdown, daemon-side.
2
+ *
3
+ * This is the renderer with its output thrown away. It parses with the same
4
+ * two extensions the app installs (`expo/components/Markdown.tsx`) and walks
5
+ * the tree with the same spec (`shared/markdown-mentions.ts`), so a handle
6
+ * fires a notification *iff* the operator would have seen it as a link.
7
+ *
8
+ * The alternative — a regex over the raw `content` — is the parallel scanner
9
+ * `docs/rendering-warnings.md` forbids for exactly this reason. Its failure is silent and
10
+ * unexplainable: a pasted log containing `@claude-4c7289d1` inside a fenced
11
+ * block shows as inert code in the UI while waking an agent, with nothing
12
+ * visible to say why.
13
+ *
14
+ * Lives here rather than in `shared/` because it imports `mdast-util-*`, and
15
+ * metro resolves a shared module's dependencies from its own directory — see
16
+ * `docs/math.md` on the `katex` alias. The daemon is the only caller.
17
+ */
18
+ import { fromMarkdown } from "mdast-util-from-markdown";
19
+ import { gfmAutolinkLiteral } from "micromark-extension-gfm-autolink-literal";
20
+ import { gfmAutolinkLiteralFromMarkdown } from "mdast-util-gfm-autolink-literal";
21
+ import { linkifyRefs } from "./shared/markdown-refs.js";
22
+ import { collectMentionSpec } from "./shared/markdown-mentions.js";
23
+ /** The narrative ids mentioned in `md`, in first-appearance order.
24
+ *
25
+ * `isKnown` is what keeps a handle that names no narrative inert — the same
26
+ * refusal `narrativeIdLinks` makes, so invented ids never become ledger rows.
27
+ * Repeats collapse: the set is the dedup within one event, the table's UNIQUE
28
+ * constraint is the dedup across re-scans. */
29
+ export function extractMentions(md, isKnown) {
30
+ if (!md.includes("@"))
31
+ return [];
32
+ const found = new Set();
33
+ const tree = fromMarkdown(md, {
34
+ extensions: [gfmAutolinkLiteral()],
35
+ mdastExtensions: [gfmAutolinkLiteralFromMarkdown()],
36
+ });
37
+ linkifyRefs(tree, collectMentionSpec(isKnown, found));
38
+ return [...found];
39
+ }
40
+ //# sourceMappingURL=mentions-scan.js.map
@@ -0,0 +1,259 @@
1
+ /**
2
+ * Confirm that the operator is at this machine, using the OS's own
3
+ * owner-authentication prompt — never one of ours.
4
+ *
5
+ * This is the human-presence check passkey enrollment rests on: a software
6
+ * authenticator can mint a passkey whose "user verified" flag is set with no
7
+ * human anywhere, and without attestation the daemon cannot tell it from
8
+ * Face ID. What an agent running as the operator cannot do is answer a
9
+ * system dialog that wants a fingerprint or the login password, so the first
10
+ * enrollment (and recovery) is confirmed here instead.
11
+ *
12
+ * Each platform runs a tool the OS already ships, so there is no native
13
+ * dependency:
14
+ *
15
+ * | platform | tool | asks for |
16
+ * |---------------|--------------------------------------|-------------------------------|
17
+ * | macOS | `osascript` → LocalAuthentication | Touch ID, Watch, or password |
18
+ * | Windows / WSL | `powershell.exe` → Windows Hello | face, fingerprint, or PIN |
19
+ * | Linux desktop | `pkcheck` → polkit (`auth_self`) | the user's own password |
20
+ *
21
+ * Headless Linux has no dialog to show and reports `unavailable`; a hardware
22
+ * key is the answer there, not a password typed into a terminal an agent can
23
+ * also read.
24
+ *
25
+ * Every command is a fixed absolute path into a system directory, never a
26
+ * bare name looked up on the daemon's inherited PATH (or a path built from
27
+ * its environment), where a careless agent's PATH tweak could put a fake
28
+ * `osascript` that prints `{"ok":true}`. That is hygiene, not a defence: a
29
+ * hostile process running as the operator can already edit `dist/` or the
30
+ * database, which brane accepts as out of scope.
31
+ *
32
+ * Each backend is a pure `PresencePlan` (command, arguments, how to read the
33
+ * answer) so the suite checks every platform's plan on any platform without
34
+ * putting a dialog on anyone's screen. Only the macOS plan has been run
35
+ * against a real prompt.
36
+ */
37
+ import { execFile } from "node:child_process";
38
+ import { readFileSync } from "node:fs";
39
+ /** The polkit action the Linux backend checks; its policy is in docs/presence.md. */
40
+ export const POLKIT_ACTION = "dev.brane.confirm-presence";
41
+ export function detectPresencePlatform(platform = process.platform, procVersion = () => readFileSync("/proc/version", "utf8"), env = process.env) {
42
+ if (platform === "darwin")
43
+ return "macos";
44
+ if (platform === "win32")
45
+ return "windows";
46
+ if (platform !== "linux")
47
+ return "unsupported";
48
+ if (env.WSL_DISTRO_NAME)
49
+ return "wsl";
50
+ try {
51
+ if (/microsoft/i.test(procVersion()))
52
+ return "wsl";
53
+ }
54
+ catch {
55
+ // no /proc: not WSL
56
+ }
57
+ return "linux";
58
+ }
59
+ // ── macOS ──────────────────────────────────────────────────────────────
60
+ /**
61
+ * JavaScript for Automation, run by `osascript -l JavaScript`. Policy 2 is
62
+ * `LAPolicyDeviceOwnerAuthentication`: biometrics, a paired Watch, or the
63
+ * login password — the dialog has no approve button of its own, so a
64
+ * synthetic click can cancel it but never pass it. The reply arrives on a
65
+ * callback, so the script spins the run loop until it does.
66
+ */
67
+ export const MAC_SCRIPT = `
68
+ ObjC.import('LocalAuthentication');
69
+ ObjC.import('Foundation');
70
+ function run(argv) {
71
+ const ctx = $.LAContext.alloc.init;
72
+ const canErr = Ref();
73
+ if (!ctx.canEvaluatePolicyError(2, canErr)) {
74
+ return JSON.stringify({ ok: false, unavailable: true, error: ObjC.unwrap(canErr[0].localizedDescription) });
75
+ }
76
+ let done = false, result = null;
77
+ ctx.evaluatePolicyLocalizedReasonReply(2, argv[0], (success, error) => {
78
+ result = success ? { ok: true } : { ok: false, code: Number(error.code), error: ObjC.unwrap(error.localizedDescription) };
79
+ done = true;
80
+ });
81
+ while (!done) $.NSRunLoop.currentRunLoop.runUntilDate($.NSDate.dateWithTimeIntervalSinceNow(0.1));
82
+ return JSON.stringify(result);
83
+ }
84
+ `;
85
+ /**
86
+ * LAError codes that mean the operator said no. Not `systemCancel` (-4: the
87
+ * system took the dialog away) and not `authenticationFailed` (-1: too many
88
+ * wrong attempts) — both are `failed`, as `RetriesExhausted` is on Windows.
89
+ */
90
+ const LA_CANCELED = new Set([-2 /* userCancel */]);
91
+ export function macPlan(reason) {
92
+ const method = "macos-local-auth";
93
+ return {
94
+ method,
95
+ command: "/usr/bin/osascript",
96
+ args: ["-l", "JavaScript", "-e", MAC_SCRIPT, reason],
97
+ interpret(stdout, stderr, code) {
98
+ let r;
99
+ try {
100
+ r = JSON.parse(stdout.trim());
101
+ }
102
+ catch {
103
+ return { ok: false, method, reason: "failed", detail: (stderr || stdout).trim() || `exit ${code}` };
104
+ }
105
+ if (r.ok === true)
106
+ return { ok: true, method };
107
+ if (r.unavailable)
108
+ return { ok: false, method, reason: "unavailable", detail: r.error };
109
+ const canceled = typeof r.code === "number" && LA_CANCELED.has(r.code);
110
+ return { ok: false, method, reason: canceled ? "canceled" : "failed", detail: r.error };
111
+ },
112
+ };
113
+ }
114
+ // ── Windows and WSL ────────────────────────────────────────────────────
115
+ /** A PowerShell single-quoted literal: only `'` is special, and it doubles. */
116
+ function psQuote(s) {
117
+ return `'${s.replace(/'/g, "''")}'`;
118
+ }
119
+ /**
120
+ * Windows Hello through `UserConsentVerifier`, from Windows PowerShell 5.1
121
+ * (`powershell.exe`; PowerShell 7 dropped the WinRT projection). From WSL
122
+ * the same file is reachable through `/mnt/c`, and the prompt appears on
123
+ * the Windows desktop, out of reach of anything inside the Linux VM.
124
+ */
125
+ export function windowsScript(reason) {
126
+ return [
127
+ "$ErrorActionPreference = 'Stop'",
128
+ "Add-Type -AssemblyName System.Runtime.WindowsRuntime",
129
+ "$null = [Windows.Security.Credentials.UI.UserConsentVerifier,Windows.Security.Credentials.UI,ContentType=WindowsRuntime]",
130
+ "$asTask = [System.WindowsRuntimeSystemExtensions].GetMethods() | Where-Object { $_.Name -eq 'AsTask' -and $_.GetParameters().Count -eq 1 -and $_.GetParameters()[0].ParameterType.Name -eq 'IAsyncOperation`1' } | Select-Object -First 1",
131
+ "function Await($op, $type) { $t = $asTask.MakeGenericMethod($type).Invoke($null, @($op)); $null = $t.Wait(-1); $t.Result }",
132
+ "$avail = Await ([Windows.Security.Credentials.UI.UserConsentVerifier]::CheckAvailabilityAsync()) ([Windows.Security.Credentials.UI.UserConsentVerifierAvailability])",
133
+ "if ($avail -ne 'Available') { Write-Output \"unavailable:$avail\"; exit 0 }",
134
+ `$r = Await ([Windows.Security.Credentials.UI.UserConsentVerifier]::RequestVerificationAsync(${psQuote(reason)})) ([Windows.Security.Credentials.UI.UserConsentVerificationResult])`,
135
+ "Write-Output \"result:$r\"",
136
+ ].join("\n");
137
+ }
138
+ export function windowsPlan(reason, platform = "windows") {
139
+ const method = "windows-hello";
140
+ // Hard-coded, not built from %SystemRoot%: the daemon inherits its
141
+ // environment, so a path read from it is only as fixed as PATH is.
142
+ const command = platform === "wsl" ? WSL_POWERSHELL : WINDOWS_POWERSHELL;
143
+ // -EncodedCommand takes UTF-16LE base64, which sidesteps every layer of
144
+ // quoting between here, WSL's interop, and the Windows command line.
145
+ const encoded = Buffer.from(windowsScript(reason), "utf16le").toString("base64");
146
+ return {
147
+ method,
148
+ command,
149
+ args: ["-NoProfile", "-NonInteractive", "-EncodedCommand", encoded],
150
+ interpret(stdout, stderr, code) {
151
+ const line = stdout.trim().split(/\r?\n/).pop() ?? "";
152
+ if (line === "result:Verified")
153
+ return { ok: true, method };
154
+ if (line.startsWith("unavailable:")) {
155
+ return { ok: false, method, reason: "unavailable", detail: line.slice("unavailable:".length) };
156
+ }
157
+ if (line === "result:Canceled") {
158
+ return { ok: false, method, reason: "canceled", detail: line.slice("result:".length) };
159
+ }
160
+ if (line.startsWith("result:"))
161
+ return { ok: false, method, reason: "failed", detail: line.slice("result:".length) };
162
+ return { ok: false, method, reason: "failed", detail: (stderr || stdout).trim() || `exit ${code}` };
163
+ },
164
+ };
165
+ }
166
+ // ── Linux ──────────────────────────────────────────────────────────────
167
+ /**
168
+ * polkit, against brane's own action. Its policy (in docs/presence.md)
169
+ * says `auth_self` — the user proves they are themselves, not an admin —
170
+ * with no retention, so every check asks. polkit's built-in actions mostly
171
+ * want an administrator, which is the wrong question, hence the file.
172
+ *
173
+ * pkcheck exit codes (pkcheck(1)): 0 authorized; 1 not authorized, which
174
+ * includes a daemon outside an active local session (over SSH, under a
175
+ * service manager), where polkit never asks; 2 a challenge but no agent to
176
+ * show it (headless); 3 dismissed, the only one that means the operator said
177
+ * no; 127 error, which is how an unregistered action is believed to surface
178
+ * (untested), so the message is matched before the code.
179
+ */
180
+ export function linuxPlan(pid = process.pid) {
181
+ const method = "polkit";
182
+ return {
183
+ method,
184
+ command: "/usr/bin/pkcheck",
185
+ args: ["--action-id", POLKIT_ACTION, "--process", String(pid), "--allow-user-interaction"],
186
+ interpret(stdout, stderr, code) {
187
+ const detail = (stderr || stdout).trim() || undefined;
188
+ if (code === 0)
189
+ return { ok: true, method };
190
+ if (/not registered|No action with action id/i.test(detail ?? "")) {
191
+ return { ok: false, method, reason: "unavailable", detail: `polkit action ${POLKIT_ACTION} is not installed` };
192
+ }
193
+ if (code === 3)
194
+ return { ok: false, method, reason: "canceled", detail };
195
+ if (code === 2)
196
+ return { ok: false, method, reason: "unavailable", detail: detail ?? "no polkit agent (headless?)" };
197
+ return { ok: false, method, reason: "failed", detail: detail ?? `exit ${code}` };
198
+ },
199
+ };
200
+ }
201
+ /** Windows PowerShell 5.1 where Windows installs it. */
202
+ export const WINDOWS_POWERSHELL = "C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\powershell.exe";
203
+ /** The same file as WSL's default automount sees it. */
204
+ export const WSL_POWERSHELL = "/mnt/c/Windows/System32/WindowsPowerShell/v1.0/powershell.exe";
205
+ // ── Dispatch ───────────────────────────────────────────────────────────
206
+ export function presencePlan(reason, platform = detectPresencePlatform()) {
207
+ switch (platform) {
208
+ case "macos":
209
+ return macPlan(reason);
210
+ case "windows":
211
+ case "wsl":
212
+ return windowsPlan(reason, platform);
213
+ case "linux":
214
+ return linuxPlan();
215
+ default:
216
+ return null;
217
+ }
218
+ }
219
+ /**
220
+ * Ask the operator to confirm they are here. `reason` is shown in the
221
+ * prompt where the platform allows it (macOS, Windows); polkit shows its
222
+ * policy's fixed message. Never throws.
223
+ */
224
+ export function confirmPresence(reason, opts = {}) {
225
+ const plan = presencePlan(reason, opts.platform);
226
+ if (!plan)
227
+ return Promise.resolve({ ok: false, method: null, reason: "unavailable", detail: process.platform });
228
+ return runPlan(plan, opts.timeoutMs ?? 120_000, opts.signal);
229
+ }
230
+ /** Run a plan's command and hand its outcome to the plan. Aborting `signal`
231
+ * kills the helper, which takes its dialog down with it on macOS (see
232
+ * docs/presence.md for Windows). Never throws. */
233
+ export function runPlan(plan, timeoutMs, signal) {
234
+ return new Promise((resolve) => {
235
+ if (signal?.aborted) {
236
+ resolve({ ok: false, method: plan.method, reason: "aborted" });
237
+ return;
238
+ }
239
+ execFile(plan.command, plan.args, { timeout: timeoutMs, maxBuffer: 1024 * 1024, windowsHide: true, signal }, (err, stdout, stderr) => {
240
+ const e = err;
241
+ if (e?.name === "AbortError") {
242
+ resolve({ ok: false, method: plan.method, reason: "aborted" });
243
+ }
244
+ else if (e?.code === "ENOENT") {
245
+ resolve({ ok: false, method: plan.method, reason: "unavailable", detail: `${plan.command} not found` });
246
+ }
247
+ else if (e?.code === "ERR_CHILD_PROCESS_STDIO_MAXBUFFER") {
248
+ resolve({ ok: false, method: plan.method, reason: "failed", detail: "output too large" });
249
+ }
250
+ else if (e?.killed) {
251
+ resolve({ ok: false, method: plan.method, reason: "timeout" });
252
+ }
253
+ else {
254
+ resolve(plan.interpret(String(stdout), String(stderr), typeof e?.code === "number" ? e.code : e ? null : 0));
255
+ }
256
+ });
257
+ });
258
+ }
259
+ //# sourceMappingURL=presence.js.map