@qvac/core 0.3.0 → 0.3.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (191) hide show
  1. package/dist-lib/lib/api/chat.d.ts +3 -1
  2. package/dist-lib/lib/api/chat.js +6 -2
  3. package/dist-lib/lib/api/index.d.ts +2 -0
  4. package/dist-lib/lib/api/index.js +3 -1
  5. package/dist-lib/lib/api/inference-calls.d.ts +18 -0
  6. package/dist-lib/lib/api/inference-calls.js +15 -0
  7. package/dist-lib/lib/api/rag.js +10 -2
  8. package/dist-lib/lib/assistant/harness.d.ts +1 -0
  9. package/dist-lib/lib/assistant/history.d.ts +10 -1
  10. package/dist-lib/lib/assistant/history.js +36 -1
  11. package/dist-lib/lib/assistant/message.d.ts +2 -0
  12. package/dist-lib/lib/assistant/message.js +11 -0
  13. package/dist-lib/lib/assistant/skill-encode.js +27 -1
  14. package/dist-lib/lib/assistant-tree.d.ts +2 -0
  15. package/dist-lib/lib/assistant-tree.js +11 -2
  16. package/dist-lib/lib/assistant.d.ts +19 -3
  17. package/dist-lib/lib/assistant.js +148 -23
  18. package/dist-lib/lib/core/chunk.js +5 -0
  19. package/dist-lib/lib/core/files/rag/build.d.ts +1 -2
  20. package/dist-lib/lib/core/files/rag/build.js +1 -2
  21. package/dist-lib/lib/core/files/rag/extract/office-collabora.d.ts +1 -1
  22. package/dist-lib/lib/core/files/rag/index.d.ts +4 -24
  23. package/dist-lib/lib/core/files/rag/index.js +3 -17
  24. package/dist-lib/lib/core/files/rag/load.d.ts +1 -2
  25. package/dist-lib/lib/core/files/rag/load.js +1 -2
  26. package/dist-lib/lib/core/files/rag/produce.d.ts +1 -1
  27. package/dist-lib/lib/core/files/rag/produce.js +1 -3
  28. package/dist-lib/lib/core/files/rag/retrieval/full-text.d.ts +1 -1
  29. package/dist-lib/lib/core/files/rag/retrieval/full-text.js +1 -4
  30. package/dist-lib/lib/core/files/rag/similarity.d.ts +5 -0
  31. package/dist-lib/lib/core/files/rag/similarity.js +6 -0
  32. package/dist-lib/lib/core/files/upload.d.ts +1 -1
  33. package/dist-lib/lib/core/inference-calls-handoff.d.ts +20 -0
  34. package/dist-lib/lib/core/inference-calls-handoff.js +37 -0
  35. package/dist-lib/lib/core/inference-calls-receiver.d.ts +14 -0
  36. package/dist-lib/lib/core/inference-calls-receiver.js +39 -0
  37. package/dist-lib/lib/core/notification.js +5 -4
  38. package/dist-lib/lib/core/skill-secrets.js +3 -6
  39. package/dist-lib/lib/core.d.ts +9 -1
  40. package/dist-lib/lib/core.js +129 -13
  41. package/dist-lib/lib/credential-pusher.d.ts +2 -3
  42. package/dist-lib/lib/credential-pusher.js +67 -39
  43. package/dist-lib/lib/inference-calls.d.ts +40 -0
  44. package/dist-lib/lib/inference-calls.js +266 -0
  45. package/dist-lib/lib/integrations/index.d.ts +1 -2
  46. package/dist-lib/lib/integrations/index.js +0 -1
  47. package/dist-lib/lib/integrations/setup-routes.js +0 -15
  48. package/dist-lib/lib/oauth/flow-core.js +1 -1
  49. package/dist-lib/lib/oauth/flow.d.ts +4 -2
  50. package/dist-lib/lib/oauth/flow.js +3 -4
  51. package/dist-lib/lib/oauth/host.d.ts +3 -1
  52. package/dist-lib/lib/oauth/host.js +3 -1
  53. package/dist-lib/lib/oauth/keys.d.ts +21 -20
  54. package/dist-lib/lib/oauth/keys.js +59 -85
  55. package/dist-lib/lib/oauth/loopback.d.ts +2 -2
  56. package/dist-lib/lib/oauth/loopback.js +32 -19
  57. package/dist-lib/lib/oauth/loopback.stub.d.ts +2 -2
  58. package/dist-lib/lib/oauth/loopback.stub.js +2 -2
  59. package/dist-lib/lib/oauth/providers.d.ts +3 -5
  60. package/dist-lib/lib/oauth/providers.js +20 -67
  61. package/dist-lib/lib/oauth/redirect-types.d.ts +1 -0
  62. package/dist-lib/lib/oauth/registry.d.ts +10 -0
  63. package/dist-lib/lib/oauth/registry.js +25 -0
  64. package/dist-lib/lib/oauth/resolve.js +1 -1
  65. package/dist-lib/lib/oauth/token-exchange-core.js +2 -1
  66. package/dist-lib/lib/oauth/types.d.ts +1 -1
  67. package/dist-lib/lib/oauth.client.d.ts +3 -1
  68. package/dist-lib/lib/oauth.client.js +3 -1
  69. package/dist-lib/lib/oauth.d.ts +4 -7
  70. package/dist-lib/lib/oauth.js +4 -7
  71. package/dist-lib/lib/skill-types.d.ts +12 -0
  72. package/dist-lib/schema/content.js +5 -1
  73. package/dist-lib/schema/engine.js +63 -2
  74. package/dist-lib/schema/qvac.js +4 -2
  75. package/dist-lib/spec/bind.js +3 -0
  76. package/dist-lib/spec/calls.js +1 -0
  77. package/dist-lib/spec/capabilities.d.ts +2 -0
  78. package/dist-lib/spec/hrpc/hrpc.json +13 -0
  79. package/dist-lib/spec/hrpc/index.js +15 -3
  80. package/dist-lib/spec/hrpc/messages.js +284 -52
  81. package/dist-lib/spec/hyperschema/index.js +284 -52
  82. package/dist-lib/spec/hyperschema/schema.json +210 -1
  83. package/dist-lib/spec/hyperschema/types.d.ts +51 -1
  84. package/dist-lib/spec/local/hyperdb/messages.js +64 -128
  85. package/dist-lib/spec/local/hyperschema/index.js +32 -64
  86. package/dist-lib/spec/local/hyperschema/schema.json +16 -0
  87. package/dist-lib/spec/mesh/hyperdb/messages.js +176 -311
  88. package/dist-lib/spec/mesh/hyperdispatch/messages.js +132 -236
  89. package/dist-lib/spec/mesh/hyperschema/index.js +132 -236
  90. package/dist-lib/spec/mesh/hyperschema/schema.json +70 -1
  91. package/dist-lib/spec/mesh/hyperschema/types.d.ts +3 -1
  92. package/package.json +15 -18
  93. package/tui/app.mjs +98 -155
  94. package/tui/lib/knowledge-import.mjs +1 -1
  95. package/tui/lib/save-file.mjs +1 -1
  96. package/.claude/skills/asana/SKILL.md +0 -86
  97. package/.claude/skills/compaction/SKILL.md +0 -136
  98. package/.claude/skills/device-mesh-debug/SKILL.md +0 -328
  99. package/.claude/skills/gh-stack/SKILL.md +0 -181
  100. package/.claude/skills/gh-stack/references/commands.md +0 -177
  101. package/.claude/skills/gh-stack/references/stack-design.md +0 -97
  102. package/.claude/skills/gh-stack/references/troubleshooting.md +0 -159
  103. package/.claude/skills/js-house-style/SKILL.md +0 -671
  104. package/.claude/skills/pr-create/SKILL.md +0 -243
  105. package/.claude/skills/pr-review/SKILL.md +0 -428
  106. package/.claude/skills/release/SKILL.md +0 -122
  107. package/CLAUDE.md +0 -48
  108. package/dist-lib/lib/core/files/rag/artifact/codec.d.ts +0 -3
  109. package/dist-lib/lib/core/files/rag/artifact/codec.js +0 -172
  110. package/dist-lib/lib/core/files/rag/artifact/id.d.ts +0 -10
  111. package/dist-lib/lib/core/files/rag/artifact/id.js +0 -39
  112. package/dist-lib/lib/core/files/rag/artifact/types.d.ts +0 -24
  113. package/dist-lib/lib/core/files/rag/artifact/types.js +0 -41
  114. package/dist-lib/lib/core/files/rag/chunk/chunking.d.ts +0 -12
  115. package/dist-lib/lib/core/files/rag/chunk/chunking.js +0 -117
  116. package/dist-lib/lib/core/files/rag/chunk/metadata.d.ts +0 -30
  117. package/dist-lib/lib/core/files/rag/chunk/metadata.js +0 -89
  118. package/dist-lib/lib/core/files/rag/chunk/tokenizer.d.ts +0 -7
  119. package/dist-lib/lib/core/files/rag/chunk/tokenizer.js +0 -18
  120. package/dist-lib/lib/core/files/rag/extract/csv-text.d.ts +0 -1
  121. package/dist-lib/lib/core/files/rag/extract/csv-text.js +0 -117
  122. package/dist-lib/lib/core/files/rag/extract/document-types.d.ts +0 -22
  123. package/dist-lib/lib/core/files/rag/extract/document-types.js +0 -152
  124. package/dist-lib/lib/core/files/rag/extract/extract.d.ts +0 -17
  125. package/dist-lib/lib/core/files/rag/extract/extract.js +0 -81
  126. package/dist-lib/lib/core/files/rag/extract/html-table-text.d.ts +0 -1
  127. package/dist-lib/lib/core/files/rag/extract/html-table-text.js +0 -125
  128. package/dist-lib/lib/core/files/rag/extract/html-text.d.ts +0 -1
  129. package/dist-lib/lib/core/files/rag/extract/html-text.js +0 -132
  130. package/dist-lib/lib/core/files/rag/extract/pdf-platform.d.ts +0 -17
  131. package/dist-lib/lib/core/files/rag/extract/pdf-platform.js +0 -141
  132. package/dist-lib/lib/core/files/rag/extract/pdf-text.d.ts +0 -12
  133. package/dist-lib/lib/core/files/rag/extract/pdf-text.js +0 -43
  134. package/dist-lib/lib/core/files/rag/extract/rtf-text.d.ts +0 -1
  135. package/dist-lib/lib/core/files/rag/extract/rtf-text.js +0 -15
  136. package/dist-lib/lib/core/files/rag/extract/text-decoding.d.ts +0 -1
  137. package/dist-lib/lib/core/files/rag/extract/text-decoding.js +0 -91
  138. package/dist-lib/lib/core/files/rag/extract/types.d.ts +0 -36
  139. package/dist-lib/lib/core/files/rag/extract/types.js +0 -12
  140. package/dist-lib/lib/core/files/rag/extract/utf.d.ts +0 -2
  141. package/dist-lib/lib/core/files/rag/extract/utf.js +0 -18
  142. package/dist-lib/lib/core/files/rag/retrieval/citation-match.d.ts +0 -2
  143. package/dist-lib/lib/core/files/rag/retrieval/citation-match.js +0 -16
  144. package/dist-lib/lib/core/files/rag/retrieval/coverage.d.ts +0 -4
  145. package/dist-lib/lib/core/files/rag/retrieval/coverage.js +0 -48
  146. package/dist-lib/lib/core/files/rag/retrieval/diversity.d.ts +0 -10
  147. package/dist-lib/lib/core/files/rag/retrieval/diversity.js +0 -37
  148. package/dist-lib/lib/core/files/rag/retrieval/floor.d.ts +0 -5
  149. package/dist-lib/lib/core/files/rag/retrieval/floor.js +0 -21
  150. package/dist-lib/lib/core/files/rag/retrieval/format.d.ts +0 -6
  151. package/dist-lib/lib/core/files/rag/retrieval/format.js +0 -21
  152. package/dist-lib/lib/core/files/rag/retrieval/knowledge.d.ts +0 -2
  153. package/dist-lib/lib/core/files/rag/retrieval/knowledge.js +0 -54
  154. package/dist-lib/lib/core/files/rag/retrieval/notices.d.ts +0 -16
  155. package/dist-lib/lib/core/files/rag/retrieval/notices.js +0 -27
  156. package/dist-lib/lib/core/files/rag/retrieval/scoring.d.ts +0 -8
  157. package/dist-lib/lib/core/files/rag/retrieval/scoring.js +0 -82
  158. package/dist-lib/lib/core/files/rag/retrieval/search.d.ts +0 -3
  159. package/dist-lib/lib/core/files/rag/retrieval/search.js +0 -49
  160. package/dist-lib/lib/core/files/rag/retrieval/selection.d.ts +0 -11
  161. package/dist-lib/lib/core/files/rag/retrieval/selection.js +0 -93
  162. package/dist-lib/lib/core/files/rag/retrieval/source-tag-filter.d.ts +0 -6
  163. package/dist-lib/lib/core/files/rag/retrieval/source-tag-filter.js +0 -79
  164. package/dist-lib/lib/core/files/rag/retrieval/sources.d.ts +0 -13
  165. package/dist-lib/lib/core/files/rag/retrieval/sources.js +0 -20
  166. package/dist-lib/lib/core/files/rag/retrieval/target.d.ts +0 -13
  167. package/dist-lib/lib/core/files/rag/retrieval/target.js +0 -89
  168. package/dist-lib/lib/core/files/rag/retrieval/terms.d.ts +0 -8
  169. package/dist-lib/lib/core/files/rag/retrieval/terms.js +0 -70
  170. package/dist-lib/lib/core/files/rag/retrieval/types.d.ts +0 -29
  171. package/dist-lib/lib/core/files/rag/retrieval/types.js +0 -1
  172. package/dist-lib/lib/core/files/rag/retrieval/web-sources.d.ts +0 -8
  173. package/dist-lib/lib/core/files/rag/retrieval/web-sources.js +0 -41
  174. package/dist-lib/lib/core/files/rag/retrieval/whole-document-query.d.ts +0 -1
  175. package/dist-lib/lib/core/files/rag/retrieval/whole-document-query.js +0 -96
  176. package/dist-lib/lib/core/files/rag/vector/memory.d.ts +0 -15
  177. package/dist-lib/lib/core/files/rag/vector/memory.js +0 -135
  178. package/dist-lib/lib/core/files/rag/vector/types.d.ts +0 -28
  179. package/dist-lib/lib/core/files/rag/vector/types.js +0 -1
  180. package/dist-lib/lib/integrations/oauth-providers.d.ts +0 -14
  181. package/dist-lib/lib/integrations/oauth-providers.js +0 -116
  182. package/dist-lib/lib/oauth/asana.d.ts +0 -35
  183. package/dist-lib/lib/oauth/asana.js +0 -71
  184. package/dist-lib/lib/oauth/google.d.ts +0 -37
  185. package/dist-lib/lib/oauth/google.js +0 -77
  186. package/dist-lib/lib/oauth/notion.d.ts +0 -34
  187. package/dist-lib/lib/oauth/notion.js +0 -82
  188. package/dist-lib/lib/oauth/spotify.d.ts +0 -32
  189. package/dist-lib/lib/oauth/spotify.js +0 -74
  190. package/dist-lib/lib/oauth/verify.d.ts +0 -8
  191. package/dist-lib/lib/oauth/verify.js +0 -58
@@ -1,122 +0,0 @@
1
- ---
2
- name: release
3
- description: Cut a release of @qvac/core — pick the version from what landed, bump package.json on a release/x.y.z branch, and roll the api changelogs from Unreleased to the dated version. Use when asked to release, cut a version, publish, or bump the version.
4
- ---
5
-
6
- # Release
7
-
8
- A release here is small and shaped by convention, and both halves matter: the
9
- version bump, and rolling `docs/api/` from `[Unreleased]` to a dated heading.
10
- Nothing else rides along.
11
-
12
- ## What publishes
13
-
14
- `.github/workflows/publish.yml` triggers on `push: tags: v*`. **The tag is the
15
- release.** It is created on the merge commit after the PR lands, never on the
16
- branch — a tag pushed from the branch publishes unreviewed code.
17
-
18
- So this skill takes you to a merged PR. Tagging is the step after, and it is the
19
- user's to take unless they ask otherwise.
20
-
21
- ## 1. Choose the version, then ask
22
-
23
- Read what has landed since the last tag:
24
-
25
- ```bash
26
- git describe --tags --abbrev=0
27
- git log --format='%s' "$(git describe --tags --abbrev=0)"..main
28
- ```
29
-
30
- Classify by conventional-commit type, and remember this package is pre-1.0:
31
-
32
- | What landed | Bump |
33
- | -------------------------------------------- | ------- |
34
- | A breaking change (`!`, or a removed export) | `minor` |
35
- | `feat` | `minor` |
36
- | `fix`, `perf`, `refactor`, `chore`, `docs` | `patch` |
37
-
38
- While the major is `0`, a breaking change is a **minor** bump, not a major one —
39
- `0.1.7 → 0.2.0` in this repo's history is exactly that. Reserve `major` for
40
- declaring the API stable, which is a decision, not a consequence of a diff.
41
-
42
- **Always suggest a version and always ask.** State the bump, the resulting
43
- version, and the one or two commits that drove it, then wait for a clear answer.
44
- The log is a signal, not the decision — a `feat` that nobody can reach yet is a
45
- patch, and a `fix` that changes a contract is not.
46
-
47
- ## 2. Branch
48
-
49
- ```bash
50
- git checkout main && git pull
51
- git checkout -b release/0.2.3
52
- ```
53
-
54
- The branch is named for the version it carries.
55
-
56
- ## 3. Bump
57
-
58
- ```bash
59
- npm version 0.2.3 --no-git-tag-version
60
- ```
61
-
62
- `--no-git-tag-version` is not optional: `npm version` tags by default, and a tag
63
- on this branch publishes before review.
64
-
65
- Then check what moved:
66
-
67
- ```bash
68
- git status --porcelain
69
- ```
70
-
71
- **Only `package.json` may change.** `package-lock.json` has been stale since
72
- `0.1.4` and every release since has left it alone; if npm touches it, revert it
73
- rather than bringing unrelated churn into a release PR. Same for `bun.lock`.
74
-
75
- ## 4. Roll the api changelogs
76
-
77
- Every file in `docs/api/` ends with a changelog in
78
- [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) form. For each file:
79
-
80
- - A `### [Unreleased]` with entries under it becomes `### [0.2.3] - 2026-09-21`,
81
- dated today. Leave the entries as they are — they were written by whoever made
82
- the change and say what a caller has to do differently.
83
- - A `### [Unreleased]` reading "No behaviour changes since …" is **not** rolled.
84
- That module did not change; giving it a dated heading would claim it did.
85
- Update its sentence to name the new version instead.
86
- - Every file's `Latest release: **0.2.2**` line becomes the new version.
87
-
88
- The next behaviour change adds `### [Unreleased]` back above the dated heading.
89
- Do not leave an empty one behind.
90
-
91
- ## 5. Commit and open
92
-
93
- The commit message and the PR title are **the version alone**:
94
-
95
- ```
96
- 0.2.3
97
- ```
98
-
99
- No type, no scope, no body. That is how every release in this log reads
100
- (`0.2.2 (#131)`, `0.2.1 (#78)`), and it is what makes them findable.
101
-
102
- A release PR carries the bump and the changelog roll and nothing else. Do not
103
- fold a fix into it, and do not fold a bump into a feature PR.
104
-
105
- Open it against `main`, not as a draft — there is nothing to iterate on.
106
-
107
- ## 6. After it merges
108
-
109
- Tag the merge commit and push the tag; that is what publishes:
110
-
111
- ```bash
112
- git checkout main && git pull
113
- git tag v0.2.3 && git push origin v0.2.3
114
- ```
115
-
116
- Confirm with the user before pushing a tag. It is the one irreversible step —
117
- npm does not let a version be republished.
118
-
119
- ## Sibling skills
120
-
121
- - Writing any other PR → **pr-create**, which also covers why release PRs are
122
- titled by version alone.
package/CLAUDE.md DELETED
@@ -1,48 +0,0 @@
1
- # @qvac/core
2
-
3
- A P2P, multi-device AI workbench core. Three pieces, one duplex stream:
4
-
5
- - **`Core`** — the state authority. Owns the replicated mesh and the device-local
6
- store, serves every capability over hrpc. It contains **no AI whatsoever**.
7
- - **`Client`** — a thin proxy whose methods are generated from the schema.
8
- Everything an app or assistant does goes through it.
9
- - **`QvacAssistant`** — the execution host. Owns the AI harness and drives it
10
- through the Client's capability surface, like any other client. **The only
11
- place inference happens.**
12
-
13
- The library runs on both Bare and Node; the test suite runs under Bare only.
14
-
15
- ## Durable invariants
16
-
17
- - **Those three roles do not blur.** AI does not move into `Core`; state
18
- authority does not move out of it; nothing reaches around `Client`.
19
- - **`spec/` is generated** by `bun run build:schema`. Never hand-edit it; CI fails
20
- when it is dirty after regeneration.
21
- - **Autobee is the source of truth** for replicated state; anything else is a
22
- materialized view.
23
- - **History is never evicted to make a prompt fit.** Compaction is the one
24
- sanctioned exception and it already exists.
25
-
26
- ## Working here
27
-
28
- - `bun run lint` (prettier + lunte + tsc) and `bun run test` (node, bare and
29
- addon lanes) before considering a change done.
30
- - Tests live in `test/` and run under brittle. A non-trivial change ships tests.
31
-
32
- ## Always
33
-
34
- - **Comment only what the code cannot say.** Why, not what. One line, never two.
35
- Most code needs none — do not narrate it. If one line cannot carry it, the code
36
- needs reshaping, not a longer comment.
37
- - **Commits read as if a human wrote them.** No `Co-Authored-By` trailer and no
38
- AI or agent attribution, in a commit message or a PR.
39
- - **Route behaviour is in `docs/api/`**, one file per api module, each with a
40
- changelog. Read the one for a route before changing it, and add to its
41
- changelog when you do. A module without a file yet gets one when a change
42
- needs it.
43
- - **Durable decisions are in `docs/decisions/`.** Read `INDEX.md` when touching an
44
- area it lists, and open a record only on a path or keyword match — they are
45
- reference, not rules. Add one when a change makes a non-obvious lasting choice.
46
- - **Branch from `main`, and open the PR against `main`.** The one exception is a
47
- stacked PR, which branches from the PR below it and targets that branch.
48
- - **Branch names never contain `codex`, `cursor` or `claude`**, in any case.
@@ -1,3 +0,0 @@
1
- import { type RagArtifactPayload } from './types.ts';
2
- export declare function encodeRagArtifact(payload: RagArtifactPayload): Buffer;
3
- export declare function decodeRagArtifact(bytes: Buffer): RagArtifactPayload;
@@ -1,172 +0,0 @@
1
- import b4a from 'b4a';
2
- import { ARTIFACT_FORMAT_VERSION, assertRagArtifactPayload } from './types.js';
3
- const MAGIC = b4a.from(ARTIFACT_FORMAT_VERSION, 'ascii');
4
- const HEADER_LENGTH_BYTES = 4;
5
- export function encodeRagArtifact(payload) {
6
- assertRagArtifactPayload(payload);
7
- assertPayloadDimensions(payload);
8
- const sections = buildSections(payload);
9
- const header = b4a.from(JSON.stringify(sections.header), 'utf8');
10
- const headerLength = b4a.alloc(HEADER_LENGTH_BYTES);
11
- b4a.writeUInt32LE(headerLength, header.byteLength, 0);
12
- return b4a.concat([
13
- MAGIC,
14
- headerLength,
15
- header,
16
- sections.text,
17
- sections.metadata,
18
- sections.embeddings
19
- ]);
20
- }
21
- export function decodeRagArtifact(bytes) {
22
- assertMagic(bytes);
23
- const headerLengthOffset = MAGIC.byteLength;
24
- const headerLength = b4a.readUInt32LE(bytes, headerLengthOffset);
25
- const headerStart = headerLengthOffset + HEADER_LENGTH_BYTES;
26
- const headerEnd = headerStart + headerLength;
27
- let parsed;
28
- try {
29
- parsed = JSON.parse(b4a.toString(bytes.subarray(headerStart, headerEnd), 'utf8'));
30
- }
31
- catch {
32
- throw new Error('RAG artifact header is not valid JSON');
33
- }
34
- const header = parseArtifactHeader(parsed);
35
- const textStart = headerEnd;
36
- const metadataStart = textStart + header.textByteLength;
37
- const embeddingStart = metadataStart + header.metadataByteLength;
38
- assertExpectedLength(bytes, header, embeddingStart);
39
- return {
40
- modelName: header.modelName,
41
- dimensions: header.dimensions,
42
- chunks: header.chunks.map((chunk, index) => decodeChunk(bytes, header, chunk, index, textStart, metadataStart, embeddingStart))
43
- };
44
- }
45
- function buildSections(payload) {
46
- const textBuffers = [];
47
- const metadataBuffers = [];
48
- const chunks = [];
49
- let contentOffset = 0;
50
- let metadataOffset = 0;
51
- for (const chunk of payload.chunks) {
52
- const content = b4a.from(chunk.content, 'utf8');
53
- const metadata = b4a.from(chunk.metadataJson, 'utf8');
54
- chunks.push({
55
- id: chunk.id,
56
- contentOffset,
57
- contentLength: content.byteLength,
58
- metadataOffset,
59
- metadataLength: metadata.byteLength
60
- });
61
- textBuffers.push(content);
62
- metadataBuffers.push(metadata);
63
- contentOffset += content.byteLength;
64
- metadataOffset += metadata.byteLength;
65
- }
66
- return {
67
- header: {
68
- modelName: payload.modelName,
69
- dimensions: payload.dimensions,
70
- chunkCount: payload.chunks.length,
71
- textByteLength: contentOffset,
72
- metadataByteLength: metadataOffset,
73
- chunks
74
- },
75
- text: b4a.concat(textBuffers),
76
- metadata: b4a.concat(metadataBuffers),
77
- embeddings: encodeEmbeddings(payload)
78
- };
79
- }
80
- function encodeEmbeddings(payload) {
81
- const embeddings = b4a.alloc(payload.chunks.length * payload.dimensions * 4);
82
- let offset = 0;
83
- for (const chunk of payload.chunks) {
84
- for (const value of chunk.embedding) {
85
- b4a.writeFloatLE(embeddings, finite(value), offset);
86
- offset += 4;
87
- }
88
- }
89
- return embeddings;
90
- }
91
- function decodeChunk(bytes, header, chunk, index, textStart, metadataStart, embeddingStart) {
92
- return {
93
- id: chunk.id,
94
- content: b4a.toString(bytes.subarray(textStart + chunk.contentOffset, textStart + chunk.contentOffset + chunk.contentLength), 'utf8'),
95
- metadataJson: b4a.toString(bytes.subarray(metadataStart + chunk.metadataOffset, metadataStart + chunk.metadataOffset + chunk.metadataLength), 'utf8'),
96
- embedding: decodeEmbedding(bytes, header.dimensions, embeddingStart, index)
97
- };
98
- }
99
- function decodeEmbedding(bytes, dimensions, embeddingStart, chunkIndex) {
100
- const values = [];
101
- let offset = embeddingStart + chunkIndex * dimensions * 4;
102
- for (let index = 0; index < dimensions; index++) {
103
- const value = b4a.readFloatLE(bytes, offset);
104
- if (!Number.isFinite(value)) {
105
- throw new Error('RAG artifact has a non-finite embedding value');
106
- }
107
- values.push(value);
108
- offset += 4;
109
- }
110
- return values;
111
- }
112
- function parseArtifactHeader(value) {
113
- const header = value;
114
- if (!header ||
115
- typeof header.modelName !== 'string' ||
116
- !isNonNegativeInteger(header.dimensions) ||
117
- header.dimensions === 0 ||
118
- !isNonNegativeInteger(header.chunkCount) ||
119
- !isNonNegativeInteger(header.textByteLength) ||
120
- !isNonNegativeInteger(header.metadataByteLength) ||
121
- !Array.isArray(header.chunks) ||
122
- !header.chunks.every(isChunkHeader)) {
123
- throw new Error('RAG artifact header is malformed');
124
- }
125
- const parsed = header;
126
- for (const chunk of parsed.chunks) {
127
- if (chunk.contentOffset + chunk.contentLength > parsed.textByteLength ||
128
- chunk.metadataOffset + chunk.metadataLength > parsed.metadataByteLength) {
129
- throw new Error('RAG artifact chunk range is out of bounds');
130
- }
131
- }
132
- return parsed;
133
- }
134
- function isChunkHeader(value) {
135
- const chunk = value;
136
- return Boolean(chunk &&
137
- typeof chunk.id === 'string' &&
138
- isNonNegativeInteger(chunk.contentOffset) &&
139
- isNonNegativeInteger(chunk.contentLength) &&
140
- isNonNegativeInteger(chunk.metadataOffset) &&
141
- isNonNegativeInteger(chunk.metadataLength));
142
- }
143
- function isNonNegativeInteger(value) {
144
- return typeof value === 'number' && Number.isInteger(value) && value >= 0;
145
- }
146
- function assertPayloadDimensions(payload) {
147
- for (const chunk of payload.chunks) {
148
- if (chunk.embedding.length !== payload.dimensions) {
149
- throw new Error(`Chunk ${chunk.id} has ${chunk.embedding.length} dimensions`);
150
- }
151
- }
152
- }
153
- function assertMagic(bytes) {
154
- if (bytes.byteLength < MAGIC.byteLength + HEADER_LENGTH_BYTES) {
155
- throw new Error('RAG artifact is truncated');
156
- }
157
- if (!b4a.equals(bytes.subarray(0, MAGIC.byteLength), MAGIC)) {
158
- throw new Error('RAG artifact has an unsupported format');
159
- }
160
- }
161
- function assertExpectedLength(bytes, header, embeddingStart) {
162
- const expected = embeddingStart + header.chunkCount * header.dimensions * 4;
163
- if (bytes.byteLength !== expected || header.chunkCount !== header.chunks.length) {
164
- throw new Error('RAG artifact length does not match its header');
165
- }
166
- }
167
- function finite(value) {
168
- if (!Number.isFinite(value)) {
169
- throw new Error('Embedding value must be finite');
170
- }
171
- return value;
172
- }
@@ -1,10 +0,0 @@
1
- export type ArtifactIdInput = {
2
- readonly attachmentId: string;
3
- readonly sourceBlobId: string;
4
- readonly sourceHash: string;
5
- readonly embeddingModel: string;
6
- };
7
- export declare function contentHash(bytes: Buffer): string;
8
- export declare function documentIndexArtifactId(input: ArtifactIdInput): string;
9
- export declare function pipelineVersionFor(model: string): string;
10
- export declare function stableHashString(value: string): string;
@@ -1,39 +0,0 @@
1
- import b4a from 'b4a';
2
- import { CHUNKER_VERSION, EMBEDDER_CODE_VERSION, EXTRACTOR_VERSION } from './types.js';
3
- export function contentHash(bytes) {
4
- return stableHashBytes(bytes);
5
- }
6
- export function documentIndexArtifactId(input) {
7
- return [
8
- 'docidx',
9
- stableHashString([
10
- input.attachmentId,
11
- input.sourceBlobId,
12
- input.sourceHash,
13
- input.embeddingModel,
14
- EMBEDDER_CODE_VERSION,
15
- EXTRACTOR_VERSION,
16
- CHUNKER_VERSION
17
- ].join('\u0000'))
18
- ].join('-');
19
- }
20
- export function pipelineVersionFor(model) {
21
- return `${model}|${EMBEDDER_CODE_VERSION}|${EXTRACTOR_VERSION}|${CHUNKER_VERSION}`;
22
- }
23
- export function stableHashString(value) {
24
- return stableHashBytes(b4a.from(value, 'utf8'));
25
- }
26
- function stableHashBytes(bytes) {
27
- let high = 0x811c9dc5;
28
- let low = 0x01000193;
29
- for (const byte of bytes) {
30
- high ^= byte;
31
- high = Math.imul(high, 0x01000193);
32
- low ^= byte + 0x9e3779b9;
33
- low = Math.imul(low, 0x85ebca6b);
34
- }
35
- return `${hex32(high)}${hex32(low)}`;
36
- }
37
- function hex32(value) {
38
- return (value >>> 0).toString(16).padStart(8, '0');
39
- }
@@ -1,24 +0,0 @@
1
- export declare const DOCUMENT_INDEX_ARTIFACT_STATUS: {
2
- readonly pending: 'pending';
3
- readonly indexing: 'indexing';
4
- readonly ready: 'ready';
5
- readonly failed: 'failed';
6
- };
7
- export type DocumentIndexArtifactStatus = (typeof DOCUMENT_INDEX_ARTIFACT_STATUS)[keyof typeof DOCUMENT_INDEX_ARTIFACT_STATUS];
8
- export declare const EXTRACTOR_VERSION = "document-extractor-v1";
9
- export declare const CHUNKER_VERSION = "rag-chunker-v1";
10
- export declare const ARTIFACT_FORMAT_VERSION = "QVRAGIDX1";
11
- export declare const EMBEDDER_CODE_VERSION = "embedder-v1";
12
- export type RagArtifactChunk = {
13
- readonly id: string;
14
- readonly content: string;
15
- readonly metadataJson: string;
16
- readonly embedding: ReadonlyArray<number>;
17
- };
18
- export type RagArtifactPayload = {
19
- readonly modelName: string;
20
- readonly dimensions: number;
21
- readonly chunks: ReadonlyArray<RagArtifactChunk>;
22
- };
23
- export declare function isDocumentIndexArtifactStatus(value: unknown): value is DocumentIndexArtifactStatus;
24
- export declare function assertRagArtifactPayload(payload: RagArtifactPayload): void;
@@ -1,41 +0,0 @@
1
- export const DOCUMENT_INDEX_ARTIFACT_STATUS = {
2
- pending: 'pending',
3
- indexing: 'indexing',
4
- ready: 'ready',
5
- failed: 'failed'
6
- };
7
- export const EXTRACTOR_VERSION = 'document-extractor-v1';
8
- export const CHUNKER_VERSION = 'rag-chunker-v1';
9
- export const ARTIFACT_FORMAT_VERSION = 'QVRAGIDX1';
10
- export const EMBEDDER_CODE_VERSION = 'embedder-v1';
11
- export function isDocumentIndexArtifactStatus(value) {
12
- return typeof value === 'string' && value in DOCUMENT_INDEX_ARTIFACT_STATUS;
13
- }
14
- export function assertRagArtifactPayload(payload) {
15
- if (typeof payload.modelName !== 'string' || payload.modelName.length === 0) {
16
- throw new Error('RAG artifact payload needs a model name');
17
- }
18
- if (!Number.isInteger(payload.dimensions) || payload.dimensions <= 0) {
19
- throw new Error('RAG artifact payload needs positive integer dimensions');
20
- }
21
- if (!Array.isArray(payload.chunks)) {
22
- throw new Error('RAG artifact payload needs a chunk array');
23
- }
24
- for (const chunk of payload.chunks) {
25
- if (chunk === null || typeof chunk !== 'object') {
26
- throw new Error('RAG artifact chunk must be an object');
27
- }
28
- if (typeof chunk.id !== 'string' || chunk.id.length === 0) {
29
- throw new Error('RAG artifact chunk needs an id');
30
- }
31
- if (typeof chunk.content !== 'string' || typeof chunk.metadataJson !== 'string') {
32
- throw new Error(`RAG artifact chunk ${chunk.id} needs string content and metadata`);
33
- }
34
- if (!Array.isArray(chunk.embedding) || !chunk.embedding.every(isNumber)) {
35
- throw new Error(`RAG artifact chunk ${chunk.id} needs a numeric embedding`);
36
- }
37
- }
38
- }
39
- function isNumber(value) {
40
- return typeof value === 'number';
41
- }
@@ -1,12 +0,0 @@
1
- import type { RagDocumentInput } from '../extract/types.ts';
2
- export type RagPreparedChunk = {
3
- readonly id: string;
4
- readonly content: string;
5
- readonly tokenCount: number;
6
- readonly metadataJson: string;
7
- };
8
- export type ChunkOptions = {
9
- readonly chunkSize?: number;
10
- readonly chunkOverlap?: number;
11
- };
12
- export declare function chunkRagDocuments(documents: ReadonlyArray<RagDocumentInput>, options: ChunkOptions): ReadonlyArray<RagPreparedChunk>;
@@ -1,117 +0,0 @@
1
- import { buildRagChunkMetadata } from './metadata.js';
2
- import { tokenizeForRag } from './tokenizer.js';
3
- const DEFAULT_CHUNK_SIZE = 256;
4
- const DEFAULT_CHUNK_OVERLAP = 50;
5
- export function chunkRagDocuments(documents, options) {
6
- return documents.flatMap((document) => chunkDocument(document, normalizeChunkOptions(options)));
7
- }
8
- function chunkDocument(document, options) {
9
- const tokens = tokenizeForRag(document.content);
10
- if (tokens.length === 0) {
11
- return [];
12
- }
13
- const chunks = [];
14
- const paragraphEnds = paragraphEndIndexes(tokens);
15
- for (let start = skipBreakTokens(tokens, 0); start < tokens.length;) {
16
- const end = chunkEnd(start, tokens.length, paragraphEnds, options.chunkSize);
17
- const first = tokens[start];
18
- const last = tokens[end - 1];
19
- if (!first || !last) {
20
- break;
21
- }
22
- const content = document.content.slice(first.start, last.end).trim();
23
- chunks.push(buildChunk(document, chunks.length, content, end - start));
24
- if (end === tokens.length) {
25
- break;
26
- }
27
- start = skipBreakTokens(tokens, nextStart(start, end, options.chunkOverlap));
28
- }
29
- return chunks;
30
- }
31
- function paragraphEndIndexes(tokens) {
32
- const ends = [];
33
- let paragraphStart = 0;
34
- let breakStart = null;
35
- let newlineCount = 0;
36
- tokens.forEach((token, index) => {
37
- const newlines = countNewlines(token.text);
38
- if (newlines === 0 && token.text.trim().length > 0) {
39
- breakStart = null;
40
- newlineCount = 0;
41
- return;
42
- }
43
- if (newlines === 0) {
44
- return;
45
- }
46
- breakStart ??= index;
47
- newlineCount += newlines;
48
- if (newlineCount < 2) {
49
- return;
50
- }
51
- if (paragraphStart < breakStart) {
52
- ends.push(breakStart);
53
- }
54
- paragraphStart = index + 1;
55
- breakStart = null;
56
- newlineCount = 0;
57
- });
58
- return [...ends, tokens.length];
59
- }
60
- function chunkEnd(start, tokenCount, paragraphEnds, chunkSize) {
61
- const maxEnd = Math.min(start + chunkSize, tokenCount);
62
- return paragraphEnds.filter((end) => end > start && end <= maxEnd).at(-1) ?? maxEnd;
63
- }
64
- function nextStart(start, end, overlap) {
65
- const overlapped = end - overlap;
66
- return overlapped > start ? overlapped : end;
67
- }
68
- function skipBreakTokens(tokens, start) {
69
- let next = start;
70
- let token = tokens[next];
71
- while (token && isParagraphSeparatorToken(token.text)) {
72
- next++;
73
- token = tokens[next];
74
- }
75
- return next;
76
- }
77
- function isParagraphSeparatorToken(text) {
78
- return text.trim().length === 0 && text.includes('\n');
79
- }
80
- function countNewlines(text) {
81
- return [...text].filter((char) => char === '\n').length;
82
- }
83
- function buildChunk(document, chunkIndex, content, tokenCount) {
84
- const documentId = document.id ?? stableDocumentId(document);
85
- return {
86
- id: chunkId(documentId, document.chatId, chunkIndex),
87
- content,
88
- tokenCount,
89
- metadataJson: JSON.stringify(buildRagChunkMetadata({ document, documentId, chunkIndex, tokenCount }))
90
- };
91
- }
92
- function chunkId(documentId, chatId, chunkIndex) {
93
- return chatId ? `${documentId}:${chatId}:${chunkIndex}` : `${documentId}:${chunkIndex}`;
94
- }
95
- function normalizeChunkOptions(options) {
96
- const chunkSize = positiveInteger(options.chunkSize, DEFAULT_CHUNK_SIZE);
97
- const requestedOverlap = nonNegativeInteger(options.chunkOverlap, DEFAULT_CHUNK_OVERLAP);
98
- const chunkOverlap = Math.min(requestedOverlap, Math.max(0, chunkSize - 1));
99
- return { chunkSize, chunkOverlap };
100
- }
101
- function positiveInteger(value, fallback) {
102
- return Number.isInteger(value) && value !== undefined && value > 0 ? value : fallback;
103
- }
104
- function nonNegativeInteger(value, fallback) {
105
- return Number.isInteger(value) && value !== undefined && value >= 0 ? value : fallback;
106
- }
107
- function stableDocumentId(document) {
108
- return `doc-${fnv1a(`${document.title}\n${document.sourcePath ?? ''}\n${document.content}`)}`;
109
- }
110
- function fnv1a(value) {
111
- let hash = 0x811c9dc5;
112
- for (let i = 0; i < value.length; i++) {
113
- hash ^= value.charCodeAt(i);
114
- hash = Math.imul(hash, 0x01000193);
115
- }
116
- return (hash >>> 0).toString(16).padStart(8, '0');
117
- }
@@ -1,30 +0,0 @@
1
- import type { RagDocumentInput } from '../extract/types.ts';
2
- export type RagChunkMetadata = {
3
- readonly documentId: string;
4
- readonly title: string;
5
- readonly sourcePath: string | null;
6
- readonly sourceUrl: string | null;
7
- readonly mimeType: string | null;
8
- readonly artifactId: string | null;
9
- readonly attachmentId: string | null;
10
- readonly messageId: string | null;
11
- readonly chatId: string | null;
12
- readonly chunkIndex: number;
13
- readonly tokenCount: number;
14
- readonly indexedAt: string;
15
- };
16
- export declare function attachmentDocumentId(attachmentId: string): string;
17
- export declare function buildRagChunkMetadata(input: {
18
- readonly document: RagDocumentInput;
19
- readonly documentId: string;
20
- readonly chunkIndex: number;
21
- readonly tokenCount: number;
22
- }): RagChunkMetadata;
23
- export declare function parseRagMetadata(metadataJson: string | null): RagChunkMetadata | null;
24
- export declare function hitMatchesFilters(hit: {
25
- readonly metadataJson: string | null;
26
- }, filters: {
27
- readonly chatId?: string;
28
- readonly documentIds?: ReadonlyArray<string>;
29
- readonly attachmentIds?: ReadonlyArray<string>;
30
- }, liveSourceIds?: ReadonlySet<string>): boolean;