cragkit 0.0.0-stage → 1.0.0-rc.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/NOTICE +22 -0
- package/dist/assessment.d.ts +10 -0
- package/dist/assessment.js +96 -0
- package/dist/canonical.d.ts +32 -0
- package/dist/canonical.js +206 -0
- package/dist/context.d.ts +25 -0
- package/dist/context.js +144 -0
- package/dist/contracts.d.ts +2 -0
- package/dist/contracts.js +2 -0
- package/dist/corrective.d.ts +252 -0
- package/dist/corrective.js +469 -0
- package/dist/embeddings.d.ts +214 -0
- package/dist/embeddings.js +296 -0
- package/dist/errors.d.ts +40 -0
- package/dist/errors.js +39 -0
- package/dist/events.d.ts +53 -0
- package/dist/events.js +166 -0
- package/dist/evidence.d.ts +10 -0
- package/dist/evidence.js +45 -0
- package/dist/execution.d.ts +42 -0
- package/dist/execution.js +157 -0
- package/dist/fusion.d.ts +87 -0
- package/dist/fusion.js +145 -0
- package/dist/hybrid.d.ts +57 -0
- package/dist/hybrid.js +141 -0
- package/dist/identifiers.d.ts +3 -0
- package/dist/identifiers.js +15 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/ingestion.d.ts +52 -0
- package/dist/ingestion.js +318 -0
- package/dist/lifecycle.d.ts +30 -0
- package/dist/lifecycle.js +194 -0
- package/dist/packing.d.ts +17 -0
- package/dist/packing.js +206 -0
- package/dist/planning.d.ts +49 -0
- package/dist/planning.js +164 -0
- package/dist/postgres-corrective.d.ts +74 -0
- package/dist/postgres-corrective.js +268 -0
- package/dist/postgres-pool.d.ts +98 -0
- package/dist/postgres-pool.js +373 -0
- package/dist/postgres-profiles.d.ts +13 -0
- package/dist/postgres-profiles.js +56 -0
- package/dist/postgres-session.d.ts +115 -0
- package/dist/postgres-session.js +751 -0
- package/dist/postgres-source.d.ts +17 -0
- package/dist/postgres-source.js +74 -0
- package/dist/postgres.d.ts +11 -0
- package/dist/postgres.js +6 -0
- package/dist/qdrant-authority.d.ts +44 -0
- package/dist/qdrant-authority.js +477 -0
- package/dist/qdrant-contract.d.ts +229 -0
- package/dist/qdrant-contract.js +288 -0
- package/dist/qdrant-corrective.d.ts +64 -0
- package/dist/qdrant-corrective.js +154 -0
- package/dist/qdrant-manifest.d.ts +29 -0
- package/dist/qdrant-manifest.js +332 -0
- package/dist/qdrant-publication-verification.d.ts +24 -0
- package/dist/qdrant-publication-verification.js +177 -0
- package/dist/qdrant-publication-worker.d.ts +90 -0
- package/dist/qdrant-publication-worker.js +978 -0
- package/dist/qdrant-publication.d.ts +42 -0
- package/dist/qdrant-publication.js +824 -0
- package/dist/qdrant-reconciliation.d.ts +48 -0
- package/dist/qdrant-reconciliation.js +414 -0
- package/dist/qdrant-recovery.d.ts +80 -0
- package/dist/qdrant-recovery.js +1924 -0
- package/dist/qdrant-security-readback.d.ts +16 -0
- package/dist/qdrant-security-readback.js +276 -0
- package/dist/qdrant-security-verification.d.ts +58 -0
- package/dist/qdrant-security-verification.js +446 -0
- package/dist/qdrant-security.d.ts +25 -0
- package/dist/qdrant-security.js +163 -0
- package/dist/qdrant-source-contract.d.ts +58 -0
- package/dist/qdrant-source-contract.js +284 -0
- package/dist/qdrant-source.d.ts +50 -0
- package/dist/qdrant-source.js +211 -0
- package/dist/qdrant-supervisor.d.ts +61 -0
- package/dist/qdrant-supervisor.js +904 -0
- package/dist/qdrant-transport.d.ts +28 -0
- package/dist/qdrant-transport.js +179 -0
- package/dist/qdrant-vectors.d.ts +24 -0
- package/dist/qdrant-vectors.js +241 -0
- package/dist/qdrant-writer-contract.d.ts +67 -0
- package/dist/qdrant-writer-contract.js +245 -0
- package/dist/qdrant-writer.d.ts +304 -0
- package/dist/qdrant-writer.js +3169 -0
- package/dist/qdrant.d.ts +38 -0
- package/dist/qdrant.js +221 -0
- package/dist/requests.d.ts +239 -0
- package/dist/requests.js +85 -0
- package/dist/reranking.d.ts +92 -0
- package/dist/reranking.js +288 -0
- package/dist/results.d.ts +366 -0
- package/dist/results.js +103 -0
- package/dist/retrieval.d.ts +13 -0
- package/dist/retrieval.js +353 -0
- package/dist/shared.d.ts +11 -0
- package/dist/shared.js +59 -0
- package/dist/vector-ingestion.d.ts +50 -0
- package/dist/vector-ingestion.js +391 -0
- package/package.json +126 -4
- package/README.md +0 -3
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Vaibhav Suman
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/NOTICE
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
Cragkit distribution notice
|
|
2
|
+
|
|
3
|
+
Cragkit's own code is distributed under the MIT License in LICENSE.
|
|
4
|
+
Copyright (c) 2026 Vaibhav Suman.
|
|
5
|
+
|
|
6
|
+
Dependencies are separate distributions. Their own licenses and notices continue
|
|
7
|
+
to apply; Cragkit's MIT declaration does not relicense them. Exact required and
|
|
8
|
+
optional versions are declared in each package manifest and the repository locks.
|
|
9
|
+
The Python core requires Pydantic; the Node core requires Zod. PostgreSQL clients,
|
|
10
|
+
HTTP clients, tokenizers, LangChain and MCP SDKs are installed only by the selected
|
|
11
|
+
profile or separate adapter package. Consult those installed distributions for
|
|
12
|
+
their license texts and transitive dependencies.
|
|
13
|
+
|
|
14
|
+
These Cragkit archives do not bundle database servers, model weights, model
|
|
15
|
+
runtimes or tokenizer rank assets. Supported server/profile names identify
|
|
16
|
+
compatibility, not a redistribution of those products. Tokenizer assets and local
|
|
17
|
+
model assets require explicit application setup and their applicable terms.
|
|
18
|
+
|
|
19
|
+
Hosted embedding and reranker profiles require an application-owned transport,
|
|
20
|
+
credential and disclosure policy. Cragkit supplies no built-in API key. Provider
|
|
21
|
+
service terms and usage charges are separate from the MIT license for this code.
|
|
22
|
+
No provider call or model download occurs merely by importing the core package.
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { type Facet } from './results.js';
|
|
2
|
+
/** Validate wire consistency and emitted references; this does not judge source truth. */
|
|
3
|
+
export declare function validate_assessment(evidence: unknown, facets: unknown): readonly Facet[];
|
|
4
|
+
export type Assessment = Readonly<{
|
|
5
|
+
facets: readonly Facet[];
|
|
6
|
+
status: 'sufficient' | 'partial' | 'insufficient';
|
|
7
|
+
policy: string;
|
|
8
|
+
}>;
|
|
9
|
+
/** Assess only exact emitted source spans against a trusted, query-bound policy. */
|
|
10
|
+
export declare function assess_evidence(query: unknown, evidence: unknown, policy?: unknown): Assessment;
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
2
|
+
import { z } from 'zod';
|
|
3
|
+
import { render_context } from './context.js';
|
|
4
|
+
import { InvalidRequest } from './errors.js';
|
|
5
|
+
import { EvidenceSchema, FacetSchema } from './results.js';
|
|
6
|
+
import { deep_freeze, identity, integer, text, valid_native_json } from './shared.js';
|
|
7
|
+
const hash = (value) => createHash('sha256').update(value, 'utf8').digest('hex');
|
|
8
|
+
const digest = z.string().regex(/^[0-9a-f]{64}$/);
|
|
9
|
+
const Anchor = z.strictObject({
|
|
10
|
+
claim_id: identity, collection: identity, document_id: identity, revision_id: identity,
|
|
11
|
+
source_sha256: digest, start_byte: integer(0, Number.MAX_SAFE_INTEGER),
|
|
12
|
+
end_byte: integer(0, Number.MAX_SAFE_INTEGER), text: text(32768, false),
|
|
13
|
+
}).refine(value => value.end_byte > value.start_byte && Buffer.byteLength(value.text) === value.end_byte - value.start_byte);
|
|
14
|
+
const PolicyFacet = z.strictObject({
|
|
15
|
+
facet_id: identity, question: text(2048), required: z.boolean(), absence: z.enum(['missing', 'unknown']),
|
|
16
|
+
anchors: z.array(Anchor).max(16).refine(values => new Set(values.map(value => JSON.stringify(value))).size === values.length),
|
|
17
|
+
});
|
|
18
|
+
const Policy = z.strictObject({
|
|
19
|
+
policy_id: identity, version: identity, query_sha256: digest, facets: z.array(PolicyFacet).min(1).max(32),
|
|
20
|
+
}).refine(value => Buffer.byteLength(`${value.policy_id}:${value.version}`) <= 256
|
|
21
|
+
&& new Set(value.facets.map(facet => facet.facet_id)).size === value.facets.length
|
|
22
|
+
&& value.facets.reduce((sum, facet) => sum + facet.anchors.length, 0) <= 64
|
|
23
|
+
&& value.facets.reduce((sum, facet) => sum + facet.anchors.reduce((bytes, anchor) => bytes + Buffer.byteLength(anchor.text), 0), 0) <= 262144);
|
|
24
|
+
function evidence_snapshot(input) {
|
|
25
|
+
render_context(input);
|
|
26
|
+
return input.map(item => EvidenceSchema.parse(item));
|
|
27
|
+
}
|
|
28
|
+
/** Validate wire consistency and emitted references; this does not judge source truth. */
|
|
29
|
+
export function validate_assessment(evidence, facets) {
|
|
30
|
+
try {
|
|
31
|
+
const items = evidence_snapshot(evidence);
|
|
32
|
+
if (!valid_native_json(facets) || !Array.isArray(facets) || facets.length > 32)
|
|
33
|
+
throw new InvalidRequest();
|
|
34
|
+
const normalized = facets.map(value => FacetSchema.parse(value));
|
|
35
|
+
const ids = new Set(items.map(item => item.evidence_id));
|
|
36
|
+
if (new Set(normalized.map(facet => facet.facet_id)).size !== normalized.length
|
|
37
|
+
|| normalized.some(facet => facet.evidence_ids.some(id => !ids.has(id))))
|
|
38
|
+
throw new InvalidRequest();
|
|
39
|
+
return deep_freeze(normalized);
|
|
40
|
+
}
|
|
41
|
+
catch {
|
|
42
|
+
throw new InvalidRequest();
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
/** Assess only exact emitted source spans against a trusted, query-bound policy. */
|
|
46
|
+
export function assess_evidence(query, evidence, policy = null) {
|
|
47
|
+
try {
|
|
48
|
+
const original = text(16384).parse(query);
|
|
49
|
+
const items = evidence_snapshot(evidence);
|
|
50
|
+
let facets, policyVersion = 'unassessed.v0';
|
|
51
|
+
if (policy === null) {
|
|
52
|
+
facets = [FacetSchema.parse({ facet_id: 'query', question: 'Original query', required: true,
|
|
53
|
+
state: 'unknown', evidence_ids: [], assessment_method: 'none', assessment_version: '0' })];
|
|
54
|
+
}
|
|
55
|
+
else {
|
|
56
|
+
if (!valid_native_json(policy))
|
|
57
|
+
throw new InvalidRequest();
|
|
58
|
+
const configured = Policy.parse(policy);
|
|
59
|
+
if (configured.query_sha256 !== hash(original))
|
|
60
|
+
throw new InvalidRequest();
|
|
61
|
+
policyVersion = `${configured.policy_id}:${configured.version}`;
|
|
62
|
+
facets = configured.facets.map(facet => {
|
|
63
|
+
const declared = new Set(facet.anchors.map(anchor => anchor.claim_id));
|
|
64
|
+
const matched = new Set(), references = [];
|
|
65
|
+
for (const item of items) {
|
|
66
|
+
let found = false;
|
|
67
|
+
for (const anchor of facet.anchors) {
|
|
68
|
+
if (item.collection !== anchor.collection || item.document_id !== anchor.document_id
|
|
69
|
+
|| item.revision_id !== anchor.revision_id || item.source_sha256 !== anchor.source_sha256
|
|
70
|
+
|| item.start_byte > anchor.start_byte || item.end_byte < anchor.end_byte)
|
|
71
|
+
continue;
|
|
72
|
+
const slice = Buffer.from(item.text).subarray(anchor.start_byte - item.start_byte, anchor.end_byte - item.start_byte);
|
|
73
|
+
if (!slice.equals(Buffer.from(anchor.text)))
|
|
74
|
+
continue;
|
|
75
|
+
matched.add(anchor.claim_id);
|
|
76
|
+
found = true;
|
|
77
|
+
}
|
|
78
|
+
if (found)
|
|
79
|
+
references.push(item.evidence_id);
|
|
80
|
+
}
|
|
81
|
+
const state = matched.size === 0 ? facet.absence
|
|
82
|
+
: matched.size >= 2 && references.length >= 2 ? 'conflicting'
|
|
83
|
+
: declared.size > 1 ? 'unknown' : 'supported';
|
|
84
|
+
return FacetSchema.parse({ facet_id: facet.facet_id, question: facet.question, required: facet.required,
|
|
85
|
+
state, evidence_ids: references, assessment_method: 'reviewed-span', assessment_version: configured.version });
|
|
86
|
+
});
|
|
87
|
+
}
|
|
88
|
+
const required = facets.filter(facet => facet.required);
|
|
89
|
+
const status = items.length === 0 ? 'insufficient'
|
|
90
|
+
: required.length > 0 && required.every(facet => facet.state === 'supported') ? 'sufficient' : 'partial';
|
|
91
|
+
return deep_freeze({ facets, status, policy: policyVersion });
|
|
92
|
+
}
|
|
93
|
+
catch {
|
|
94
|
+
throw new InvalidRequest();
|
|
95
|
+
}
|
|
96
|
+
}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
export type CanonicalOptions = Readonly<{
|
|
2
|
+
format?: string;
|
|
3
|
+
max_source_bytes?: number;
|
|
4
|
+
target_bytes?: number;
|
|
5
|
+
max_chunk_bytes?: number;
|
|
6
|
+
token_counter?: ((text: string) => number) | null;
|
|
7
|
+
tokenizer_id?: string | null;
|
|
8
|
+
token_target?: number | null;
|
|
9
|
+
max_tokens?: number | null;
|
|
10
|
+
}>;
|
|
11
|
+
export type CanonicalChunk = Readonly<{
|
|
12
|
+
chunk_id: string;
|
|
13
|
+
ordinal: number;
|
|
14
|
+
start_byte: number;
|
|
15
|
+
end_byte: number;
|
|
16
|
+
text: string;
|
|
17
|
+
text_sha256: string;
|
|
18
|
+
oversized_split: boolean;
|
|
19
|
+
token_count: number | null;
|
|
20
|
+
}>;
|
|
21
|
+
export type CanonicalDocument = Readonly<{
|
|
22
|
+
canonical_text: string;
|
|
23
|
+
source_sha256: string;
|
|
24
|
+
canonical_sha256: string;
|
|
25
|
+
parser_id: string;
|
|
26
|
+
chunker_profile: string;
|
|
27
|
+
format: string;
|
|
28
|
+
tokenizer_id: string | null;
|
|
29
|
+
token_count: number | null;
|
|
30
|
+
chunks: readonly CanonicalChunk[];
|
|
31
|
+
}>;
|
|
32
|
+
export declare function canonicalize(input: string | Uint8Array, options?: CanonicalOptions): CanonicalDocument;
|
|
@@ -0,0 +1,206 @@
|
|
|
1
|
+
/** Bounded native text normalization and exact canonical UTF-8 chunk spans. */
|
|
2
|
+
import { createHash } from 'node:crypto';
|
|
3
|
+
import { InvalidRequest } from './errors.js';
|
|
4
|
+
const hash = (value) => createHash('sha256').update(value).digest('hex');
|
|
5
|
+
const blank = (value) => !/[^ \t\r\n\f\v]/.test(value);
|
|
6
|
+
const scalar = (value) => !/[\uD800-\uDFFF]/u.test(value);
|
|
7
|
+
const integer = (value, min, max) => typeof value === 'number' && Number.isSafeInteger(value) && value >= min && value <= max;
|
|
8
|
+
const bytes = (value) => Buffer.byteLength(value, 'utf8');
|
|
9
|
+
function structural_blocks(text, markdown) {
|
|
10
|
+
const lines = text.match(/[^\n]*\n|[^\n]+$/g) ?? [];
|
|
11
|
+
const output = [];
|
|
12
|
+
const blankLine = (line) => /^[ \t]*(?:\n)?$/.test(line);
|
|
13
|
+
const heading = (line) => /^ {0,3}#{1,6}(?:[ \t]|\n|$)/.test(line);
|
|
14
|
+
const fence = (line) => /^ {0,3}(`{3,}|~{3,})/.exec(line);
|
|
15
|
+
let index = 0;
|
|
16
|
+
while (index < lines.length) {
|
|
17
|
+
if (blankLine(lines[index])) {
|
|
18
|
+
const start = index;
|
|
19
|
+
while (index < lines.length && blankLine(lines[index]))
|
|
20
|
+
index++;
|
|
21
|
+
const value = lines.slice(start, index).join('');
|
|
22
|
+
if (output.length)
|
|
23
|
+
output[output.length - 1] += value;
|
|
24
|
+
else
|
|
25
|
+
output.push(value);
|
|
26
|
+
continue;
|
|
27
|
+
}
|
|
28
|
+
const start = index;
|
|
29
|
+
const opening = markdown ? fence(lines[index]) : null;
|
|
30
|
+
if (opening) {
|
|
31
|
+
index++;
|
|
32
|
+
const closing = new RegExp('^ {0,3}' + opening[1][0] + '{' + opening[1].length + ',}[ \\t]*(?:\\n|$)');
|
|
33
|
+
while (index < lines.length) {
|
|
34
|
+
const closed = closing.test(lines[index]);
|
|
35
|
+
index++;
|
|
36
|
+
if (closed)
|
|
37
|
+
break;
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
else if (markdown && heading(lines[index]))
|
|
41
|
+
index++;
|
|
42
|
+
else {
|
|
43
|
+
index++;
|
|
44
|
+
while (index < lines.length && !blankLine(lines[index]) && !(markdown && (heading(lines[index]) || fence(lines[index]))))
|
|
45
|
+
index++;
|
|
46
|
+
}
|
|
47
|
+
while (index < lines.length && blankLine(lines[index]))
|
|
48
|
+
index++;
|
|
49
|
+
output.push(lines.slice(start, index).join(''));
|
|
50
|
+
}
|
|
51
|
+
return output;
|
|
52
|
+
}
|
|
53
|
+
export function canonicalize(input, options = {}) {
|
|
54
|
+
try {
|
|
55
|
+
return canonicalize_checked(input, options);
|
|
56
|
+
}
|
|
57
|
+
catch {
|
|
58
|
+
throw new InvalidRequest();
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
function canonicalize_checked(input, options) {
|
|
62
|
+
if (!options || typeof options !== 'object' || ![Object.prototype, null].includes(Object.getPrototypeOf(options)))
|
|
63
|
+
throw new InvalidRequest();
|
|
64
|
+
const allowed = new Set(['format', 'max_source_bytes', 'target_bytes', 'max_chunk_bytes', 'token_counter', 'tokenizer_id', 'token_target', 'max_tokens']);
|
|
65
|
+
for (const key of Reflect.ownKeys(options)) {
|
|
66
|
+
const descriptor = Object.getOwnPropertyDescriptor(options, key);
|
|
67
|
+
if (typeof key !== 'string' || !allowed.has(key) || !('value' in descriptor) || !descriptor.enumerable)
|
|
68
|
+
throw new InvalidRequest();
|
|
69
|
+
}
|
|
70
|
+
const format = options.format === undefined ? 'text/plain' : options.format;
|
|
71
|
+
if (typeof format !== 'string')
|
|
72
|
+
throw new InvalidRequest();
|
|
73
|
+
if (!['text/plain', 'text/markdown'].includes(format))
|
|
74
|
+
throw new InvalidRequest();
|
|
75
|
+
const maximum = options.max_source_bytes === undefined ? 1_048_576 : options.max_source_bytes;
|
|
76
|
+
const target = options.target_bytes === undefined ? 2048 : options.target_bytes;
|
|
77
|
+
const hard = options.max_chunk_bytes === undefined ? 32768 : options.max_chunk_bytes;
|
|
78
|
+
if (!integer(maximum, 1, 16_777_216) || !integer(hard, 4, 32768) || !integer(target, 1, hard))
|
|
79
|
+
throw new InvalidRequest();
|
|
80
|
+
const tokenValues = [options.token_counter, options.tokenizer_id, options.token_target, options.max_tokens];
|
|
81
|
+
const tokenMode = tokenValues.some(value => value !== undefined && value !== null);
|
|
82
|
+
if (tokenMode && (typeof options.token_counter !== 'function' || typeof options.tokenizer_id !== 'string'
|
|
83
|
+
|| !/^[A-Za-z0-9_.:-]{1,64}$/.test(options.tokenizer_id) || !integer(options.token_target, 1, 1_000_000)
|
|
84
|
+
|| !integer(options.max_tokens, 1, 1_000_000) || options.token_target > options.max_tokens))
|
|
85
|
+
throw new InvalidRequest();
|
|
86
|
+
let raw;
|
|
87
|
+
if (typeof input === 'string') {
|
|
88
|
+
if (input.length > maximum || !scalar(input) || bytes(input) > maximum)
|
|
89
|
+
throw new InvalidRequest();
|
|
90
|
+
raw = Buffer.from(input, 'utf8');
|
|
91
|
+
}
|
|
92
|
+
else if (input instanceof Uint8Array) {
|
|
93
|
+
if (input.byteLength > maximum)
|
|
94
|
+
throw new InvalidRequest();
|
|
95
|
+
raw = Uint8Array.from(input);
|
|
96
|
+
}
|
|
97
|
+
else
|
|
98
|
+
throw new InvalidRequest();
|
|
99
|
+
const source_sha256 = hash(raw);
|
|
100
|
+
let text = new TextDecoder('utf-8', { fatal: true, ignoreBOM: true }).decode(raw);
|
|
101
|
+
if (text.startsWith('\uFEFF'))
|
|
102
|
+
text = text.slice(1);
|
|
103
|
+
text = text.replace(/\r\n?/g, '\n');
|
|
104
|
+
if (!scalar(text) || text.includes('\0') || blank(text))
|
|
105
|
+
throw new InvalidRequest();
|
|
106
|
+
const canonical_sha256 = hash(text);
|
|
107
|
+
const profile = tokenMode
|
|
108
|
+
? `cragkit.structure.tokens.v1:target_bytes=${target}:max_bytes=${hard}:tokenizer=${options.tokenizer_id}:target=${options.token_target}:max=${options.max_tokens}`
|
|
109
|
+
: `cragkit.structure.bytes.v1:target=${target}:max=${hard}`;
|
|
110
|
+
let counterCalls = 0;
|
|
111
|
+
const count = (value) => {
|
|
112
|
+
if (!tokenMode)
|
|
113
|
+
return null;
|
|
114
|
+
if (++counterCalls > 65536)
|
|
115
|
+
throw new InvalidRequest();
|
|
116
|
+
let result;
|
|
117
|
+
try {
|
|
118
|
+
result = options.token_counter(value);
|
|
119
|
+
}
|
|
120
|
+
catch {
|
|
121
|
+
throw new InvalidRequest();
|
|
122
|
+
}
|
|
123
|
+
if (result instanceof Promise) {
|
|
124
|
+
// Invalid asynchronous counters must not leave an unhandled rejection behind.
|
|
125
|
+
void result.catch(() => { });
|
|
126
|
+
throw new InvalidRequest();
|
|
127
|
+
}
|
|
128
|
+
if (!integer(result, 0, Number.MAX_SAFE_INTEGER))
|
|
129
|
+
throw new InvalidRequest();
|
|
130
|
+
return result;
|
|
131
|
+
};
|
|
132
|
+
const chunks = [];
|
|
133
|
+
let offset = 0;
|
|
134
|
+
const emit = (value, oversized) => {
|
|
135
|
+
if (chunks.length >= 1024)
|
|
136
|
+
throw new InvalidRequest();
|
|
137
|
+
const end = offset + bytes(value);
|
|
138
|
+
const token_count = count(value);
|
|
139
|
+
if (tokenMode && token_count > options.max_tokens)
|
|
140
|
+
throw new InvalidRequest();
|
|
141
|
+
chunks.push(Object.freeze({ chunk_id: hash(`${canonical_sha256}:${offset}:${end}:${profile}`), ordinal: chunks.length,
|
|
142
|
+
start_byte: offset, end_byte: end, text: value, text_sha256: hash(value), oversized_split: oversized, token_count }));
|
|
143
|
+
offset = end;
|
|
144
|
+
};
|
|
145
|
+
let pending = [];
|
|
146
|
+
let pendingBytes = 0;
|
|
147
|
+
let pendingSplit = false;
|
|
148
|
+
const flush = () => {
|
|
149
|
+
if (pending.length)
|
|
150
|
+
emit(pending.join(''), pendingSplit);
|
|
151
|
+
pending = [];
|
|
152
|
+
pendingBytes = 0;
|
|
153
|
+
pendingSplit = false;
|
|
154
|
+
};
|
|
155
|
+
const pack = (value, split) => {
|
|
156
|
+
const valueBytes = bytes(value);
|
|
157
|
+
if (pending.length) {
|
|
158
|
+
// Match Python's work budget: impossible byte combinations never call a tokenizer.
|
|
159
|
+
if (pendingBytes + valueBytes <= target
|
|
160
|
+
&& (!tokenMode || count(pending.join('') + value) <= options.token_target)) {
|
|
161
|
+
pending.push(value);
|
|
162
|
+
pendingBytes += valueBytes;
|
|
163
|
+
pendingSplit ||= split;
|
|
164
|
+
return;
|
|
165
|
+
}
|
|
166
|
+
flush();
|
|
167
|
+
}
|
|
168
|
+
pending = [value];
|
|
169
|
+
pendingBytes = valueBytes;
|
|
170
|
+
pendingSplit = split;
|
|
171
|
+
};
|
|
172
|
+
for (const block of structural_blocks(text, format === 'text/markdown')) {
|
|
173
|
+
const blockCount = count(block);
|
|
174
|
+
if (bytes(block) > hard || (tokenMode && blockCount > options.max_tokens)) {
|
|
175
|
+
// Only oversized structures are split; every split ends at a Unicode scalar boundary.
|
|
176
|
+
let piece = '';
|
|
177
|
+
let pieceBytes = 0;
|
|
178
|
+
for (const point of block) {
|
|
179
|
+
const combined = piece + point;
|
|
180
|
+
const pointBytes = bytes(point);
|
|
181
|
+
const combinedBytes = pieceBytes + pointBytes;
|
|
182
|
+
const candidateCount = count(combined);
|
|
183
|
+
if (piece && (combinedBytes > target || combinedBytes > hard || (tokenMode && (candidateCount > options.token_target || candidateCount > options.max_tokens)))) {
|
|
184
|
+
pack(piece, true);
|
|
185
|
+
piece = point;
|
|
186
|
+
pieceBytes = pointBytes;
|
|
187
|
+
if (pieceBytes > hard || (tokenMode && count(piece) > options.max_tokens))
|
|
188
|
+
throw new InvalidRequest();
|
|
189
|
+
}
|
|
190
|
+
else {
|
|
191
|
+
if (combinedBytes > hard || (tokenMode && candidateCount > options.max_tokens))
|
|
192
|
+
throw new InvalidRequest();
|
|
193
|
+
piece = combined;
|
|
194
|
+
pieceBytes = combinedBytes;
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
if (piece)
|
|
198
|
+
pack(piece, true);
|
|
199
|
+
}
|
|
200
|
+
else
|
|
201
|
+
pack(block, false);
|
|
202
|
+
}
|
|
203
|
+
flush();
|
|
204
|
+
return Object.freeze({ canonical_text: text, source_sha256, canonical_sha256, parser_id: format === 'text/markdown' ? 'cragkit.markdown.v1' : 'cragkit.text.v1',
|
|
205
|
+
chunker_profile: profile, format, tokenizer_id: tokenMode ? options.tokenizer_id : null, token_count: count(text), chunks: Object.freeze(chunks) });
|
|
206
|
+
}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
export declare const CONTEXT_RENDERER = "cragkit.context.v2";
|
|
2
|
+
export declare class Cl100kCounter {
|
|
3
|
+
#private;
|
|
4
|
+
readonly tokenizer = "cl100k_base";
|
|
5
|
+
private constructor();
|
|
6
|
+
static create(path: string | URL): Promise<Cl100kCounter>;
|
|
7
|
+
close(): void;
|
|
8
|
+
count(value: unknown): number;
|
|
9
|
+
}
|
|
10
|
+
export declare function render_context(input: unknown): string;
|
|
11
|
+
export type ContextOptions = {
|
|
12
|
+
max_tokens: number;
|
|
13
|
+
reserve?: number;
|
|
14
|
+
counter?: Cl100kCounter | null;
|
|
15
|
+
counting_mode?: 'strict' | 'estimated';
|
|
16
|
+
};
|
|
17
|
+
export type CountedContext = Readonly<{
|
|
18
|
+
context: string;
|
|
19
|
+
context_tokens: number;
|
|
20
|
+
total_tokens: number;
|
|
21
|
+
renderer: typeof CONTEXT_RENDERER;
|
|
22
|
+
tokenizer: 'cl100k_base' | null;
|
|
23
|
+
counting_mode: 'strict' | 'estimated';
|
|
24
|
+
}>;
|
|
25
|
+
export declare function count_context(input: unknown, options: ContextOptions): CountedContext;
|
package/dist/context.js
ADDED
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
2
|
+
import { open } from 'node:fs/promises';
|
|
3
|
+
import { ContextBudgetTooSmall, InvalidRequest, UnsupportedCapability } from './errors.js';
|
|
4
|
+
import { EvidenceSchema } from './results.js';
|
|
5
|
+
import { valid_native_json } from './shared.js';
|
|
6
|
+
export const CONTEXT_RENDERER = 'cragkit.context.v2';
|
|
7
|
+
const issued = new WeakSet();
|
|
8
|
+
const construction = Symbol();
|
|
9
|
+
const SHA256 = '223921b76ee99bde995b7ff738513eef100fb51d18c93597a113bcffe865b2a7';
|
|
10
|
+
const PATTERN = String.raw `'(?i:[sdmt]|ll|ve|re)|[^\r\n\p{L}\p{N}]?+\p{L}++|\p{N}{1,3}+| ?[^\s\p{L}\p{N}]++[\r\n]*+|\s++$|\s*[\r\n]|\s+(?!\S)|\s`;
|
|
11
|
+
export class Cl100kCounter {
|
|
12
|
+
tokenizer = 'cl100k_base';
|
|
13
|
+
#encoder;
|
|
14
|
+
#closed = false;
|
|
15
|
+
constructor(encoder, token) { if (token !== construction)
|
|
16
|
+
throw new UnsupportedCapability(); this.#encoder = encoder; issued.add(this); Object.freeze(this); }
|
|
17
|
+
static async create(path) {
|
|
18
|
+
let ranks;
|
|
19
|
+
try {
|
|
20
|
+
if (typeof path !== 'string' && !(path instanceof URL))
|
|
21
|
+
throw new InvalidRequest();
|
|
22
|
+
const stream = await open(path, 'r');
|
|
23
|
+
try {
|
|
24
|
+
if (!(await stream.stat()).isFile())
|
|
25
|
+
throw new InvalidRequest();
|
|
26
|
+
const buffer = Buffer.alloc(2_000_001);
|
|
27
|
+
let size = 0;
|
|
28
|
+
while (size < buffer.length) {
|
|
29
|
+
const { bytesRead } = await stream.read(buffer, size, buffer.length - size, size);
|
|
30
|
+
if (bytesRead === 0)
|
|
31
|
+
break;
|
|
32
|
+
size += bytesRead;
|
|
33
|
+
}
|
|
34
|
+
const data = buffer.subarray(0, size);
|
|
35
|
+
if (size > 2_000_000 || createHash('sha256').update(data).digest('hex') !== SHA256)
|
|
36
|
+
throw new InvalidRequest();
|
|
37
|
+
ranks = data.toString('ascii');
|
|
38
|
+
}
|
|
39
|
+
finally {
|
|
40
|
+
await stream.close();
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
catch {
|
|
44
|
+
throw new InvalidRequest();
|
|
45
|
+
}
|
|
46
|
+
try {
|
|
47
|
+
const name = 'tiktoken/lite';
|
|
48
|
+
const module = await import(name);
|
|
49
|
+
return new Cl100kCounter(new module.Tiktoken(ranks, {}, PATTERN), construction);
|
|
50
|
+
}
|
|
51
|
+
catch {
|
|
52
|
+
throw new UnsupportedCapability();
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
close() { if (!this.#closed) {
|
|
56
|
+
this.#encoder.free();
|
|
57
|
+
this.#closed = true;
|
|
58
|
+
issued.delete(this);
|
|
59
|
+
} }
|
|
60
|
+
count(value) {
|
|
61
|
+
if (this.#closed)
|
|
62
|
+
throw new UnsupportedCapability();
|
|
63
|
+
if (typeof value !== 'string' || value.length > 262144 || !valid_native_json(value) || Buffer.byteLength(value, 'utf8') > 262144)
|
|
64
|
+
throw new InvalidRequest();
|
|
65
|
+
try {
|
|
66
|
+
return this.#encoder.encode(value, [], []).length;
|
|
67
|
+
}
|
|
68
|
+
catch {
|
|
69
|
+
throw new UnsupportedCapability();
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
export function render_context(input) {
|
|
74
|
+
if (!Array.isArray(input) || input.length > 64 || !valid_native_json(input))
|
|
75
|
+
throw new InvalidRequest();
|
|
76
|
+
try {
|
|
77
|
+
const items = input.map(value => EvidenceSchema.parse(value));
|
|
78
|
+
const ids = new Set(), digests = new Map();
|
|
79
|
+
const ordered = items.map(item => {
|
|
80
|
+
if (ids.has(item.evidence_id))
|
|
81
|
+
throw new InvalidRequest();
|
|
82
|
+
ids.add(item.evidence_id);
|
|
83
|
+
const key = JSON.stringify([item.collection, item.document_id, item.revision_id]);
|
|
84
|
+
if (digests.has(key) && digests.get(key) !== item.source_sha256)
|
|
85
|
+
throw new InvalidRequest();
|
|
86
|
+
digests.set(key, item.source_sha256);
|
|
87
|
+
return {
|
|
88
|
+
evidence_id: item.evidence_id, collection: item.collection, document_id: item.document_id,
|
|
89
|
+
revision_id: item.revision_id, chunk_id: item.chunk_id, source_locator: item.source_locator,
|
|
90
|
+
start_byte: item.start_byte, end_byte: item.end_byte, text: item.text,
|
|
91
|
+
source_sha256: item.source_sha256, text_sha256: item.text_sha256,
|
|
92
|
+
};
|
|
93
|
+
});
|
|
94
|
+
const rendered = JSON.stringify(ordered);
|
|
95
|
+
if (Buffer.byteLength(rendered, 'utf8') > 262144)
|
|
96
|
+
throw new InvalidRequest();
|
|
97
|
+
return rendered;
|
|
98
|
+
}
|
|
99
|
+
catch {
|
|
100
|
+
throw new InvalidRequest();
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
export function count_context(input, options) {
|
|
104
|
+
if (!options || typeof options !== 'object' || Object.getPrototypeOf(options) !== Object.prototype
|
|
105
|
+
|| Object.getOwnPropertySymbols(options).length)
|
|
106
|
+
throw new InvalidRequest();
|
|
107
|
+
const fields = Object.getOwnPropertyDescriptors(options);
|
|
108
|
+
if (Object.entries(fields).some(([key, value]) => !['max_tokens', 'reserve', 'counter', 'counting_mode'].includes(key)
|
|
109
|
+
|| !('value' in value) || !value.enumerable))
|
|
110
|
+
throw new InvalidRequest();
|
|
111
|
+
const maximum = fields.max_tokens?.value;
|
|
112
|
+
const reserve = fields.reserve ? fields.reserve.value : 256;
|
|
113
|
+
const mode = fields.counting_mode ? fields.counting_mode.value : 'estimated';
|
|
114
|
+
if (typeof maximum !== 'number' || !Number.isInteger(maximum) || maximum < 1 || maximum > 1000000
|
|
115
|
+
|| typeof reserve !== 'number' || !Number.isInteger(reserve) || reserve < 0 || reserve > 1000000
|
|
116
|
+
|| (mode !== 'strict' && mode !== 'estimated'))
|
|
117
|
+
throw new InvalidRequest();
|
|
118
|
+
const counter = fields.counter?.value;
|
|
119
|
+
if (mode === 'estimated' && fields.counter && counter !== null)
|
|
120
|
+
throw new InvalidRequest();
|
|
121
|
+
if (mode === 'strict' && (!(counter instanceof Cl100kCounter) || !issued.has(counter)))
|
|
122
|
+
throw new UnsupportedCapability();
|
|
123
|
+
const count = (value) => {
|
|
124
|
+
if (mode === 'estimated')
|
|
125
|
+
return Math.ceil(Buffer.byteLength(value, 'utf8') / 4);
|
|
126
|
+
try {
|
|
127
|
+
const result = Cl100kCounter.prototype.count.call(counter, value);
|
|
128
|
+
if (!Number.isSafeInteger(result) || result < 0)
|
|
129
|
+
throw new UnsupportedCapability();
|
|
130
|
+
return result;
|
|
131
|
+
}
|
|
132
|
+
catch {
|
|
133
|
+
throw new UnsupportedCapability();
|
|
134
|
+
}
|
|
135
|
+
};
|
|
136
|
+
const context = render_context(input);
|
|
137
|
+
if (count('[]') + reserve > maximum)
|
|
138
|
+
throw new ContextBudgetTooSmall();
|
|
139
|
+
const context_tokens = count(context);
|
|
140
|
+
if (context_tokens + reserve > maximum)
|
|
141
|
+
throw new ContextBudgetTooSmall();
|
|
142
|
+
return Object.freeze({ context, context_tokens, total_tokens: context_tokens + reserve,
|
|
143
|
+
renderer: CONTEXT_RENDERER, tokenizer: mode === 'strict' ? 'cl100k_base' : null, counting_mode: mode });
|
|
144
|
+
}
|