@gmickel/gno 2.12.2 → 2.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/assets/skill/SKILL.md +36 -5
- package/assets/skill/cli-reference.md +62 -11
- package/assets/skill/examples.md +30 -4
- package/assets/skill/mcp-reference.md +16 -0
- package/assets/spa-production.json.gz +0 -0
- package/browser-extension/artifacts/{gno-browser-clipper-v2.12.2.zip → gno-browser-clipper-v2.14.0.zip} +0 -0
- package/browser-extension/artifacts/gno-browser-clipper-v2.14.0.zip.sha256 +1 -0
- package/browser-extension/dist/manifest.json +1 -1
- package/package.json +1 -1
- package/spec/cli.md +139 -16
- package/spec/mcp.md +111 -3
- package/spec/output-schemas/collection-list.schema.json +28 -0
- package/spec/output-schemas/convert.schema.json +97 -0
- package/spec/output-schemas/ls.schema.json +65 -0
- package/spec/output-schemas/mcp-job-status.schema.json +17 -1
- package/spec/output-schemas/mcp-sync-result.schema.json +9 -2
- package/spec/output-schemas/search-results.schema.json +7 -0
- package/src/cli/commands/index-cmd.ts +50 -12
- package/src/cli/commands/ls.ts +20 -3
- package/src/cli/commands/profile-apply.ts +2 -1
- package/src/cli/commands/query.ts +21 -8
- package/src/cli/commands/search.ts +11 -0
- package/src/cli/commands/vsearch.ts +17 -4
- package/src/cli/options.ts +19 -0
- package/src/cli/program.ts +90 -10
- package/src/config/loader.ts +11 -1
- package/src/config/types.ts +90 -1
- package/src/core/audit-collection-rules.ts +331 -0
- package/src/core/audit-links.ts +3 -1
- package/src/core/audit-workspace.ts +69 -5
- package/src/core/collection-egress-policy-service.ts +4 -1
- package/src/core/collection-scope.ts +105 -0
- package/src/core/egress-enforcement.ts +20 -4
- package/src/core/egress-policy.ts +4 -0
- package/src/core/egress-provenance.ts +33 -11
- package/src/core/errors.ts +5 -0
- package/src/core/job-manager.ts +238 -19
- package/src/core/project-profile-apply-state.ts +6 -15
- package/src/core/project-profile-apply.ts +32 -1
- package/src/core/reindex-paths.ts +167 -0
- package/src/core/retrieval-trace-request.ts +1 -0
- package/src/ingestion/convert-file.ts +181 -0
- package/src/llm/registry.ts +20 -0
- package/src/mcp/http-egress.ts +21 -2
- package/src/mcp/tools/collection-scope.ts +25 -0
- package/src/mcp/tools/index-cmd.ts +66 -48
- package/src/mcp/tools/index.ts +74 -10
- package/src/mcp/tools/job-status.ts +7 -0
- package/src/mcp/tools/query.ts +28 -7
- package/src/mcp/tools/search.ts +4 -0
- package/src/mcp/tools/sync.ts +68 -60
- package/src/mcp/tools/vsearch.ts +9 -3
- package/src/pipeline/filters.ts +48 -5
- package/src/pipeline/graph-retrieval.ts +1 -0
- package/src/pipeline/hybrid.ts +15 -2
- package/src/pipeline/hydration.ts +7 -2
- package/src/pipeline/owner-fusion.ts +1 -0
- package/src/pipeline/search.ts +2 -0
- package/src/pipeline/types.ts +7 -0
- package/src/pipeline/vsearch.ts +10 -5
- package/src/sdk/client.ts +45 -12
- package/src/sdk/convert.ts +27 -0
- package/src/sdk/documents.ts +15 -2
- package/src/sdk/index.ts +2 -0
- package/src/sdk/types.ts +4 -1
- package/src/serve/jobs.ts +4 -1
- package/src/serve/public/components/IndexingProgress.tsx +25 -3
- package/src/serve/routes/api.ts +164 -24
- package/src/store/sqlite/adapter.ts +97 -27
- package/src/store/sqlite/eligibility.ts +4 -0
- package/src/store/types.ts +7 -0
- package/src/store/vector/sqlite-vec.ts +67 -15
- package/src/store/vector/variant-search.ts +41 -4
- package/browser-extension/artifacts/gno-browser-clipper-v2.12.2.zip.sha256 +0 -1
package/src/serve/routes/api.ts
CHANGED
|
@@ -16,7 +16,6 @@ import type {
|
|
|
16
16
|
EgressPolicy,
|
|
17
17
|
ModelPreset,
|
|
18
18
|
} from "../../config/types";
|
|
19
|
-
import type { JobManager } from "../../core/job-manager";
|
|
20
19
|
import type { ProjectAffinityScoringInput } from "../../pipeline/project-affinity";
|
|
21
20
|
import type {
|
|
22
21
|
AskResult,
|
|
@@ -57,6 +56,7 @@ import {
|
|
|
57
56
|
type EgressRelaxationConfirmation,
|
|
58
57
|
} from "../../core/collection-egress-policy-service";
|
|
59
58
|
import { parsePolicySetBody } from "../../core/collection-egress-policy-validation";
|
|
59
|
+
import { resolveCollectionScope } from "../../core/collection-scope";
|
|
60
60
|
import {
|
|
61
61
|
type ConnectorVerificationCode,
|
|
62
62
|
getConnectorVerificationRemediation,
|
|
@@ -82,6 +82,7 @@ import {
|
|
|
82
82
|
planCreateFolder,
|
|
83
83
|
planDuplicateRefactor,
|
|
84
84
|
} from "../../core/file-refactors";
|
|
85
|
+
import { JobError, type JobManager } from "../../core/job-manager";
|
|
85
86
|
import {
|
|
86
87
|
MemoryError,
|
|
87
88
|
type MemoryErrorCode,
|
|
@@ -111,6 +112,7 @@ import {
|
|
|
111
112
|
resolveRemoteProjectAffinity,
|
|
112
113
|
} from "../../core/project-affinity-surface";
|
|
113
114
|
import { projectRecordEvidenceMetadata } from "../../core/record-metadata";
|
|
115
|
+
import { resolveReindexScope } from "../../core/reindex-paths";
|
|
114
116
|
import {
|
|
115
117
|
isLeaseBusy,
|
|
116
118
|
localRequestLedger,
|
|
@@ -303,6 +305,8 @@ export interface SearchRequestBody {
|
|
|
303
305
|
limit?: number;
|
|
304
306
|
minScore?: number;
|
|
305
307
|
collection?: string;
|
|
308
|
+
/** Collection allowlist (1-256 names); not combined with `collection`. */
|
|
309
|
+
collections?: string[];
|
|
306
310
|
intent?: string;
|
|
307
311
|
exclude?: string;
|
|
308
312
|
since?: string;
|
|
@@ -323,6 +327,8 @@ export interface QueryRequestBody {
|
|
|
323
327
|
limit?: number;
|
|
324
328
|
minScore?: number;
|
|
325
329
|
collection?: string;
|
|
330
|
+
/** Collection allowlist (1-256 names); not combined with `collection`. */
|
|
331
|
+
collections?: string[];
|
|
326
332
|
lang?: string;
|
|
327
333
|
intent?: string;
|
|
328
334
|
candidateLimit?: number;
|
|
@@ -496,6 +502,7 @@ export interface ImportPreviewRequestBody {
|
|
|
496
502
|
export interface SyncRequestBody {
|
|
497
503
|
collection?: string;
|
|
498
504
|
gitPull?: boolean;
|
|
505
|
+
paths?: string[];
|
|
499
506
|
}
|
|
500
507
|
|
|
501
508
|
export interface CreateDocRequestBody {
|
|
@@ -651,6 +658,42 @@ function parseRestMetadataFilter(
|
|
|
651
658
|
return { ok: true, filter: normalizeMetadataPredicate(parsed.data) };
|
|
652
659
|
}
|
|
653
660
|
|
|
661
|
+
/**
|
|
662
|
+
* Resolve a search/query `collections` allowlist: 400 VALIDATION for malformed
|
|
663
|
+
* input or `collection` with `collections`, 404 NOT_FOUND for unknown names.
|
|
664
|
+
*/
|
|
665
|
+
async function parseRestCollectionScope(
|
|
666
|
+
body: { collection?: unknown; collections?: unknown },
|
|
667
|
+
configured: readonly { name: string }[] | null,
|
|
668
|
+
store: SqliteAdapter
|
|
669
|
+
): Promise<
|
|
670
|
+
| { ok: true; collections: string[] | undefined }
|
|
671
|
+
| { ok: false; response: Response }
|
|
672
|
+
> {
|
|
673
|
+
if (body.collections === undefined)
|
|
674
|
+
return { ok: true, collections: undefined };
|
|
675
|
+
let names = configured;
|
|
676
|
+
if (!names) {
|
|
677
|
+
const stored = await store.getCollections();
|
|
678
|
+
if (!stored.ok) {
|
|
679
|
+
return {
|
|
680
|
+
ok: false,
|
|
681
|
+
response: errorResponse("RUNTIME", stored.error.message, 500),
|
|
682
|
+
};
|
|
683
|
+
}
|
|
684
|
+
names = stored.value;
|
|
685
|
+
}
|
|
686
|
+
const scope = resolveCollectionScope(body, names);
|
|
687
|
+
if (scope.ok) return { ok: true, collections: scope.collections };
|
|
688
|
+
return {
|
|
689
|
+
ok: false,
|
|
690
|
+
response:
|
|
691
|
+
scope.kind === "validation"
|
|
692
|
+
? errorResponse("VALIDATION", scope.message)
|
|
693
|
+
: errorResponse("NOT_FOUND", scope.message, 404),
|
|
694
|
+
};
|
|
695
|
+
}
|
|
696
|
+
|
|
654
697
|
function fileRefactorHttpErrorResponse(error: FileRefactorHttpError): Response {
|
|
655
698
|
return errorResponse(error.code, error.message, error.status, error.details);
|
|
656
699
|
}
|
|
@@ -1780,6 +1823,9 @@ export async function handleSync(
|
|
|
1780
1823
|
if (body.gitPull !== undefined && typeof body.gitPull !== "boolean") {
|
|
1781
1824
|
return errorResponse("VALIDATION", "gitPull must be a boolean");
|
|
1782
1825
|
}
|
|
1826
|
+
if (body.paths !== undefined && !Array.isArray(body.paths)) {
|
|
1827
|
+
return errorResponse("VALIDATION", "paths must be an array of strings");
|
|
1828
|
+
}
|
|
1783
1829
|
|
|
1784
1830
|
// Get collections to sync (case-insensitive matching)
|
|
1785
1831
|
const collectionName = body.collection?.toLowerCase();
|
|
@@ -1801,31 +1847,95 @@ export async function handleSync(
|
|
|
1801
1847
|
return errorResponse("VALIDATION", "No collections to sync");
|
|
1802
1848
|
}
|
|
1803
1849
|
|
|
1804
|
-
|
|
1805
|
-
|
|
1806
|
-
|
|
1807
|
-
|
|
1808
|
-
const result = await defaultSyncService.syncAll(
|
|
1809
|
-
collections,
|
|
1810
|
-
store,
|
|
1811
|
-
withContentTypeRules(
|
|
1812
|
-
{
|
|
1813
|
-
gitPull: body.gitPull,
|
|
1814
|
-
runUpdateCmd: true,
|
|
1815
|
-
},
|
|
1816
|
-
ctxHolder.config
|
|
1817
|
-
)
|
|
1818
|
-
);
|
|
1819
|
-
recordContentMutation(result, ctxHolder.markContentMutation);
|
|
1820
|
-
if (hasContentMutation(result)) {
|
|
1821
|
-
if (ctxHolder.scheduler) {
|
|
1822
|
-
await ctxHolder.scheduler.triggerNow();
|
|
1823
|
-
}
|
|
1824
|
-
}
|
|
1825
|
-
return result;
|
|
1850
|
+
const options = withContentTypeRules(
|
|
1851
|
+
{
|
|
1852
|
+
gitPull: body.gitPull,
|
|
1853
|
+
runUpdateCmd: true,
|
|
1826
1854
|
},
|
|
1827
|
-
ctxHolder.
|
|
1855
|
+
ctxHolder.config
|
|
1828
1856
|
);
|
|
1857
|
+
let paths: string[] | null;
|
|
1858
|
+
try {
|
|
1859
|
+
// A path-level sync runs no update command; only git pull conflicts.
|
|
1860
|
+
paths = await resolveReindexScope(
|
|
1861
|
+
body.collection ? (collections[0] ?? null) : null,
|
|
1862
|
+
body.paths,
|
|
1863
|
+
{ gitPull: body.gitPull }
|
|
1864
|
+
);
|
|
1865
|
+
} catch (error) {
|
|
1866
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1867
|
+
const separator = message.indexOf(": ");
|
|
1868
|
+
const code = separator > 0 ? message.slice(0, separator) : "";
|
|
1869
|
+
return errorResponse(
|
|
1870
|
+
code === "INVALID_PATH" ? "INVALID_PATH" : "VALIDATION",
|
|
1871
|
+
separator > 0 ? message.slice(separator + 2) : message
|
|
1872
|
+
);
|
|
1873
|
+
}
|
|
1874
|
+
|
|
1875
|
+
// Captured now: a queued pass runs after this request's context is gone.
|
|
1876
|
+
const config = ctxHolder.config;
|
|
1877
|
+
const runSync = async (
|
|
1878
|
+
scopedPaths: string[] | null,
|
|
1879
|
+
passOptions: { gitPull?: boolean; runUpdateCmd?: boolean }
|
|
1880
|
+
): Promise<SyncResult> => {
|
|
1881
|
+
const collection = collections[0];
|
|
1882
|
+
const result =
|
|
1883
|
+
scopedPaths && collection
|
|
1884
|
+
? wrapCollectionSync(
|
|
1885
|
+
await defaultSyncService.syncPaths(
|
|
1886
|
+
collection,
|
|
1887
|
+
store,
|
|
1888
|
+
scopedPaths,
|
|
1889
|
+
withContentTypeRules({}, config)
|
|
1890
|
+
)
|
|
1891
|
+
)
|
|
1892
|
+
: await defaultSyncService.syncAll(
|
|
1893
|
+
collections,
|
|
1894
|
+
store,
|
|
1895
|
+
withContentTypeRules({ ...passOptions }, config)
|
|
1896
|
+
);
|
|
1897
|
+
recordContentMutation(result, ctxHolder.markContentMutation);
|
|
1898
|
+
if (hasContentMutation(result)) {
|
|
1899
|
+
if (ctxHolder.scheduler) {
|
|
1900
|
+
await ctxHolder.scheduler.triggerNow();
|
|
1901
|
+
}
|
|
1902
|
+
}
|
|
1903
|
+
return result;
|
|
1904
|
+
};
|
|
1905
|
+
|
|
1906
|
+
const manager = ctxHolder.jobManager;
|
|
1907
|
+
if (manager) {
|
|
1908
|
+
try {
|
|
1909
|
+
const accepted = await manager.submitReindex({
|
|
1910
|
+
type: "sync",
|
|
1911
|
+
collection: body.collection ? (collections[0]?.name ?? null) : null,
|
|
1912
|
+
paths,
|
|
1913
|
+
// A whole sync over REST always runs the collection's update command.
|
|
1914
|
+
options: { gitPull: body.gitPull ?? false, runUpdateCmd: true },
|
|
1915
|
+
run: async (scope) => ({
|
|
1916
|
+
kind: "sync",
|
|
1917
|
+
value: await runSync(scope.paths, scope.options),
|
|
1918
|
+
}),
|
|
1919
|
+
});
|
|
1920
|
+
return jsonResponse(
|
|
1921
|
+
{ jobId: accepted.jobId, status: accepted.status },
|
|
1922
|
+
202
|
|
1923
|
+
);
|
|
1924
|
+
} catch (error) {
|
|
1925
|
+
if (error instanceof JobError && error.code === "QUEUE_FULL") {
|
|
1926
|
+
return errorResponse("QUEUE_FULL", error.message, 429);
|
|
1927
|
+
}
|
|
1928
|
+
if (error instanceof JobError) {
|
|
1929
|
+
return errorResponse("CONFLICT", error.message, 409, {
|
|
1930
|
+
activeJobId: manager.getActiveJob()?.id ?? "",
|
|
1931
|
+
});
|
|
1932
|
+
}
|
|
1933
|
+
throw error;
|
|
1934
|
+
}
|
|
1935
|
+
}
|
|
1936
|
+
|
|
1937
|
+
// Standalone fallback (unit fixtures without a resident job manager).
|
|
1938
|
+
const jobResult = startJob("sync", () => runSync(paths, options));
|
|
1829
1939
|
|
|
1830
1940
|
if (!jobResult.ok) {
|
|
1831
1941
|
return jobConflictResponse(jobResult);
|
|
@@ -1834,6 +1944,20 @@ export async function handleSync(
|
|
|
1834
1944
|
return jsonResponse({ jobId: jobResult.jobId }, 202);
|
|
1835
1945
|
}
|
|
1836
1946
|
|
|
1947
|
+
function wrapCollectionSync(
|
|
1948
|
+
result: Awaited<ReturnType<typeof defaultSyncService.syncPaths>>
|
|
1949
|
+
): SyncResult {
|
|
1950
|
+
return {
|
|
1951
|
+
collections: [result],
|
|
1952
|
+
totalDurationMs: result.durationMs,
|
|
1953
|
+
totalFilesProcessed: result.filesProcessed,
|
|
1954
|
+
totalFilesAdded: result.filesAdded,
|
|
1955
|
+
totalFilesUpdated: result.filesUpdated,
|
|
1956
|
+
totalFilesErrored: result.filesErrored,
|
|
1957
|
+
totalFilesSkipped: result.filesSkipped,
|
|
1958
|
+
};
|
|
1959
|
+
}
|
|
1960
|
+
|
|
1837
1961
|
/**
|
|
1838
1962
|
* GET /api/docs
|
|
1839
1963
|
* Query params: collection, limit (default 20), offset (default 0), tagsAll, tagsAny
|
|
@@ -4587,6 +4711,13 @@ export async function handleSearch(
|
|
|
4587
4711
|
const parsedFilter = parseRestMetadataFilter(body.filter);
|
|
4588
4712
|
if (!parsedFilter.ok) return parsedFilter.response;
|
|
4589
4713
|
const filter = parsedFilter.filter;
|
|
4714
|
+
const collectionScope = await parseRestCollectionScope(
|
|
4715
|
+
body,
|
|
4716
|
+
context?.config.collections ?? null,
|
|
4717
|
+
store
|
|
4718
|
+
);
|
|
4719
|
+
if (!collectionScope.ok) return collectionScope.response;
|
|
4720
|
+
const { collections } = collectionScope;
|
|
4590
4721
|
let projectAffinity: ProjectAffinityScoringInput | undefined;
|
|
4591
4722
|
try {
|
|
4592
4723
|
projectAffinity = context
|
|
@@ -4606,6 +4737,7 @@ export async function handleSearch(
|
|
|
4606
4737
|
limit: Math.min(body.limit || 10, 50),
|
|
4607
4738
|
minScore: body.minScore,
|
|
4608
4739
|
collection: body.collection,
|
|
4740
|
+
collections,
|
|
4609
4741
|
intent: body.intent?.trim() || undefined,
|
|
4610
4742
|
exclude,
|
|
4611
4743
|
tagsAll,
|
|
@@ -4785,6 +4917,13 @@ export async function handleQuery(
|
|
|
4785
4917
|
const parsedFilter = parseRestMetadataFilter(body.filter);
|
|
4786
4918
|
if (!parsedFilter.ok) return parsedFilter.response;
|
|
4787
4919
|
const filter = parsedFilter.filter;
|
|
4920
|
+
const collectionScope = await parseRestCollectionScope(
|
|
4921
|
+
body,
|
|
4922
|
+
ctx.config.collections,
|
|
4923
|
+
ctx.store
|
|
4924
|
+
);
|
|
4925
|
+
if (!collectionScope.ok) return collectionScope.response;
|
|
4926
|
+
const { collections } = collectionScope;
|
|
4788
4927
|
let projectAffinity: ProjectAffinityScoringInput | undefined;
|
|
4789
4928
|
try {
|
|
4790
4929
|
projectAffinity = await resolveRemoteProjectAffinity(
|
|
@@ -4804,6 +4943,7 @@ export async function handleQuery(
|
|
|
4804
4943
|
limit: Math.min(body.limit ?? 20, 50),
|
|
4805
4944
|
minScore: body.minScore,
|
|
4806
4945
|
collection: body.collection,
|
|
4946
|
+
collections,
|
|
4807
4947
|
lang: body.lang,
|
|
4808
4948
|
intent: body.intent?.trim() || undefined,
|
|
4809
4949
|
candidateLimit:
|
|
@@ -164,7 +164,7 @@ import {
|
|
|
164
164
|
import { getSchemaVersion, migrations, runMigrations } from "../migrations";
|
|
165
165
|
import { err, ok } from "../types";
|
|
166
166
|
import { getStoredEmbeddingFingerprint } from "../vector/freshness";
|
|
167
|
-
import { modelTableName } from "../vector/sqlite-vec";
|
|
167
|
+
import { bindReadConnection, modelTableName } from "../vector/sqlite-vec";
|
|
168
168
|
import {
|
|
169
169
|
getVariantStatus,
|
|
170
170
|
listVectorPartitions,
|
|
@@ -622,7 +622,12 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
|
|
|
622
622
|
private activeTransaction?: {
|
|
623
623
|
token: { revoked: boolean };
|
|
624
624
|
release: () => void;
|
|
625
|
+
db: Database;
|
|
625
626
|
};
|
|
627
|
+
/** Set while a read-only caller runs; its reads use `readDb`. */
|
|
628
|
+
private readonly readScope = new AsyncLocalStorage<true>();
|
|
629
|
+
/** Second connection for read-only callers, opened on first use. */
|
|
630
|
+
private readDb: Database | null = null;
|
|
626
631
|
private shutdownFenced = false;
|
|
627
632
|
private shutdownDeadline?: number;
|
|
628
633
|
private contextGeneration = 0;
|
|
@@ -741,6 +746,8 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
|
|
|
741
746
|
|
|
742
747
|
async close(): Promise<void> {
|
|
743
748
|
this.fenceForShutdown();
|
|
749
|
+
this.readDb?.close(false);
|
|
750
|
+
this.readDb = null;
|
|
744
751
|
if (this.db) {
|
|
745
752
|
try {
|
|
746
753
|
// Current Bun finalizes all statements and releases Windows handles now.
|
|
@@ -791,7 +798,7 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
|
|
|
791
798
|
// JS cannot interleave this synchronous rollback with a running callback.
|
|
792
799
|
// A callback suspended at await must never commit or use the store again.
|
|
793
800
|
transaction.token.revoked = true;
|
|
794
|
-
|
|
801
|
+
transaction.db.exec("ROLLBACK");
|
|
795
802
|
transaction.release();
|
|
796
803
|
this.activeTransaction = undefined;
|
|
797
804
|
}
|
|
@@ -800,6 +807,47 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
|
|
|
800
807
|
return this.db !== null;
|
|
801
808
|
}
|
|
802
809
|
|
|
810
|
+
/**
|
|
811
|
+
* Run a read-only caller on a second connection to the same index. All
|
|
812
|
+
* writes stay on the shared connection, so a write transaction there (a
|
|
813
|
+
* sync batch held open across file conversions) is never visible to these
|
|
814
|
+
* reads: WAL gives this connection the last committed state. An in-memory
|
|
815
|
+
* index has no second connection and reads on the shared one.
|
|
816
|
+
*/
|
|
817
|
+
withReadConnection<T>(fn: () => Promise<T>): Promise<T> {
|
|
818
|
+
if (this.dbPath === "" || this.dbPath === ":memory:") return fn();
|
|
819
|
+
return this.readScope.run(true, fn);
|
|
820
|
+
}
|
|
821
|
+
|
|
822
|
+
private openReadDb(main: Database): Database {
|
|
823
|
+
if (this.readDb) return this.readDb;
|
|
824
|
+
const db = new Database(this.dbPath);
|
|
825
|
+
try {
|
|
826
|
+
// A write here would wait out the busy timeout while the shared
|
|
827
|
+
// connection's batch, on the same event loop, cannot commit: fail fast.
|
|
828
|
+
db.exec("PRAGMA query_only = ON");
|
|
829
|
+
db.exec("PRAGMA foreign_keys = ON");
|
|
830
|
+
const busyTimeout =
|
|
831
|
+
main.query<{ timeout: number }, []>("PRAGMA busy_timeout").get()
|
|
832
|
+
?.timeout ?? DEFAULT_BUSY_TIMEOUT_MS;
|
|
833
|
+
db.exec(`PRAGMA busy_timeout = ${busyTimeout}`);
|
|
834
|
+
if (this.ftsTokenizer.startsWith("snowball")) {
|
|
835
|
+
const snowball = loadFts5Snowball(db);
|
|
836
|
+
if (!snowball.loaded)
|
|
837
|
+
throw new Error(`Failed to load fts5-snowball: ${snowball.error}`);
|
|
838
|
+
}
|
|
839
|
+
} catch (cause) {
|
|
840
|
+
db.close(false);
|
|
841
|
+
throw cause;
|
|
842
|
+
}
|
|
843
|
+
bindReadConnection(db, main, async () => {
|
|
844
|
+
const release = await this.acquireTransactionWriter();
|
|
845
|
+
release();
|
|
846
|
+
});
|
|
847
|
+
this.readDb = db;
|
|
848
|
+
return db;
|
|
849
|
+
}
|
|
850
|
+
|
|
803
851
|
/**
|
|
804
852
|
* Run an async function within a single SQLite transaction.
|
|
805
853
|
* Uses SAVEPOINT for nesting safety.
|
|
@@ -808,6 +856,9 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
|
|
|
808
856
|
* explicit BEGIN/COMMIT to support async callbacks.
|
|
809
857
|
*/
|
|
810
858
|
async withTransaction<T>(fn: () => Promise<T>): Promise<StoreResult<T>> {
|
|
859
|
+
// Writes always run on the shared connection, even from a read scope.
|
|
860
|
+
if (this.readScope.getStore())
|
|
861
|
+
return this.readScope.exit(() => this.withTransaction(fn));
|
|
811
862
|
const parent = this.txContext.getStore();
|
|
812
863
|
const connection = this.ensureOpen();
|
|
813
864
|
const isOuter = parent === undefined;
|
|
@@ -826,7 +877,7 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
|
|
|
826
877
|
if (isOuter) {
|
|
827
878
|
// IMMEDIATE reduces lock churn for bulk writes
|
|
828
879
|
db.exec("BEGIN IMMEDIATE");
|
|
829
|
-
this.activeTransaction = { token, release: releaseWriter
|
|
880
|
+
this.activeTransaction = { token, release: releaseWriter!, db };
|
|
830
881
|
} else {
|
|
831
882
|
db.exec(`SAVEPOINT ${savepoint}`);
|
|
832
883
|
}
|
|
@@ -872,6 +923,11 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
|
|
|
872
923
|
}
|
|
873
924
|
}
|
|
874
925
|
|
|
926
|
+
/** Write outside a transaction, on the shared connection even from a read scope. */
|
|
927
|
+
private async admitWrite<T>(write: (db: Database) => T): Promise<T> {
|
|
928
|
+
return this.readScope.exit(() => write(this.ensureOpen()));
|
|
929
|
+
}
|
|
930
|
+
|
|
875
931
|
private async acquireTransactionWriter(): Promise<() => void> {
|
|
876
932
|
const previous = this.txTail;
|
|
877
933
|
let release!: () => void;
|
|
@@ -913,8 +969,12 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
|
|
|
913
969
|
if (!this.db) {
|
|
914
970
|
throw new Error("Database not open");
|
|
915
971
|
}
|
|
916
|
-
|
|
917
|
-
|
|
972
|
+
const db =
|
|
973
|
+
this.readScope.getStore() && !this.txContext.getStore()
|
|
974
|
+
? this.openReadDb(this.db)
|
|
975
|
+
: this.db;
|
|
976
|
+
this.capShutdownBusyWait(db);
|
|
977
|
+
return db;
|
|
918
978
|
}
|
|
919
979
|
|
|
920
980
|
// ─────────────────────────────────────────────────────────────────────────
|
|
@@ -1507,14 +1567,16 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
|
|
|
1507
1567
|
async createRetrievalTrace(
|
|
1508
1568
|
input: RetrievalTraceInput
|
|
1509
1569
|
): Promise<StoreResult<RetrievalTraceAppendResult>> {
|
|
1510
|
-
return
|
|
1570
|
+
return this.admitWrite((db) => createStoredTrace(db, input));
|
|
1511
1571
|
}
|
|
1512
1572
|
|
|
1513
1573
|
async mergeRetrievalTraceEgressLineage(
|
|
1514
1574
|
traceId: string,
|
|
1515
1575
|
lineage: EgressLineage
|
|
1516
1576
|
): Promise<StoreResult<RetrievalTraceAppendResult>> {
|
|
1517
|
-
return
|
|
1577
|
+
return this.admitWrite((db) =>
|
|
1578
|
+
mergeStoredTraceEgressLineage(db, traceId, lineage)
|
|
1579
|
+
);
|
|
1518
1580
|
}
|
|
1519
1581
|
|
|
1520
1582
|
async getRetrievalTrace(
|
|
@@ -1542,37 +1604,39 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
|
|
|
1542
1604
|
status: RetrievalTraceTerminalStatus,
|
|
1543
1605
|
updatedAtMs: number
|
|
1544
1606
|
): Promise<StoreResult<RetrievalTraceAppendResult>> {
|
|
1545
|
-
return
|
|
1607
|
+
return this.admitWrite((db) =>
|
|
1608
|
+
finalizeStoredTrace(db, traceId, status, updatedAtMs)
|
|
1609
|
+
);
|
|
1546
1610
|
}
|
|
1547
1611
|
|
|
1548
1612
|
async appendRetrievalTraceRun(
|
|
1549
1613
|
input: RetrievalTraceRunInput
|
|
1550
1614
|
): Promise<StoreResult<RetrievalTraceAppendResult>> {
|
|
1551
|
-
return
|
|
1615
|
+
return this.admitWrite((db) => appendStoredTraceRun(db, input));
|
|
1552
1616
|
}
|
|
1553
1617
|
|
|
1554
1618
|
async appendRetrievalTraceEvent(
|
|
1555
1619
|
input: RetrievalTraceEventInput
|
|
1556
1620
|
): Promise<StoreResult<RetrievalTraceAppendResult>> {
|
|
1557
|
-
return
|
|
1621
|
+
return this.admitWrite((db) => appendStoredTraceEvent(db, input));
|
|
1558
1622
|
}
|
|
1559
1623
|
|
|
1560
1624
|
async appendRetrievalTraceJudgment(
|
|
1561
1625
|
input: RetrievalTraceJudgmentInput
|
|
1562
1626
|
): Promise<StoreResult<RetrievalTraceAppendResult>> {
|
|
1563
|
-
return
|
|
1627
|
+
return this.admitWrite((db) => appendStoredTraceJudgment(db, input));
|
|
1564
1628
|
}
|
|
1565
1629
|
|
|
1566
1630
|
async appendRetrievalTraceExport(
|
|
1567
1631
|
input: RetrievalTraceExportInput
|
|
1568
1632
|
): Promise<StoreResult<RetrievalTraceAppendResult>> {
|
|
1569
|
-
return
|
|
1633
|
+
return this.admitWrite((db) => appendStoredTraceExport(db, input));
|
|
1570
1634
|
}
|
|
1571
1635
|
|
|
1572
1636
|
async appendRetrievalTraceExportManifest(
|
|
1573
1637
|
input: RetrievalTraceExportManifestInput
|
|
1574
1638
|
): Promise<StoreResult<RetrievalTraceAppendResult>> {
|
|
1575
|
-
return
|
|
1639
|
+
return this.admitWrite((db) => appendStoredTraceExportManifest(db, input));
|
|
1576
1640
|
}
|
|
1577
1641
|
|
|
1578
1642
|
async getRetrievalTraceExportManifest(
|
|
@@ -1590,32 +1654,34 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
|
|
|
1590
1654
|
async getOrCreateRetrievalTraceRedactionSecret(): Promise<
|
|
1591
1655
|
StoreResult<string>
|
|
1592
1656
|
> {
|
|
1593
|
-
return
|
|
1657
|
+
return this.admitWrite((db) => getOrCreateStoredTraceRedactionSecret(db));
|
|
1594
1658
|
}
|
|
1595
1659
|
|
|
1596
1660
|
async deleteRetrievalTrace(
|
|
1597
1661
|
traceId: string
|
|
1598
1662
|
): Promise<StoreResult<RetrievalTraceDeleteCounts>> {
|
|
1599
|
-
return
|
|
1663
|
+
return this.admitWrite((db) => deleteStoredTrace(db, traceId));
|
|
1600
1664
|
}
|
|
1601
1665
|
|
|
1602
1666
|
async purgeRetrievalTraces(): Promise<
|
|
1603
1667
|
StoreResult<RetrievalTracePurgeResult>
|
|
1604
1668
|
> {
|
|
1605
|
-
return
|
|
1669
|
+
return this.admitWrite((db) => purgeStoredTraces(db));
|
|
1606
1670
|
}
|
|
1607
1671
|
|
|
1608
1672
|
async enforceRetrievalTraceRetention(
|
|
1609
1673
|
policy: RetrievalTraceRetentionPolicy,
|
|
1610
1674
|
nowMs: number
|
|
1611
1675
|
): Promise<StoreResult<RetrievalTraceRetentionResult>> {
|
|
1612
|
-
return
|
|
1676
|
+
return this.admitWrite((db) =>
|
|
1677
|
+
enforceStoredTraceRetention(db, policy, nowMs)
|
|
1678
|
+
);
|
|
1613
1679
|
}
|
|
1614
1680
|
|
|
1615
1681
|
async appendEgressAuditReceipt(
|
|
1616
1682
|
receipt: EgressAuditReceiptInput
|
|
1617
1683
|
): Promise<StoreResult<RetrievalTraceAppendResult>> {
|
|
1618
|
-
return
|
|
1684
|
+
return this.admitWrite((db) => appendStoredEgressAuditReceipt(db, receipt));
|
|
1619
1685
|
}
|
|
1620
1686
|
|
|
1621
1687
|
async appendEgressAuditReceiptWithRetention(
|
|
@@ -1623,11 +1689,8 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
|
|
|
1623
1689
|
policy: EgressAuditRetentionPolicy,
|
|
1624
1690
|
nowMs: number
|
|
1625
1691
|
): Promise<StoreResult<RetrievalTraceAppendResult>> {
|
|
1626
|
-
return
|
|
1627
|
-
|
|
1628
|
-
receipt,
|
|
1629
|
-
policy,
|
|
1630
|
-
nowMs
|
|
1692
|
+
return this.admitWrite((db) =>
|
|
1693
|
+
appendStoredEgressAuditReceiptWithRetention(db, receipt, policy, nowMs)
|
|
1631
1694
|
);
|
|
1632
1695
|
}
|
|
1633
1696
|
|
|
@@ -1647,7 +1710,7 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
|
|
|
1647
1710
|
async deleteEgressAuditReceipt(
|
|
1648
1711
|
auditId: string
|
|
1649
1712
|
): Promise<StoreResult<EgressAuditDeleteResult>> {
|
|
1650
|
-
return
|
|
1713
|
+
return this.admitWrite((db) => deleteStoredEgressAuditReceipt(db, auditId));
|
|
1651
1714
|
}
|
|
1652
1715
|
|
|
1653
1716
|
async getEgressAuditStatus(): Promise<StoreResult<EgressAuditStatusResult>> {
|
|
@@ -1658,13 +1721,15 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
|
|
|
1658
1721
|
policy: EgressAuditRetentionPolicy,
|
|
1659
1722
|
nowMs: number
|
|
1660
1723
|
): Promise<StoreResult<EgressAuditRetentionResult>> {
|
|
1661
|
-
return
|
|
1724
|
+
return this.admitWrite((db) =>
|
|
1725
|
+
enforceStoredEgressAuditRetention(db, policy, nowMs)
|
|
1726
|
+
);
|
|
1662
1727
|
}
|
|
1663
1728
|
|
|
1664
1729
|
async purgeEgressAuditReceipts(): Promise<
|
|
1665
1730
|
StoreResult<EgressAuditPurgeResult>
|
|
1666
1731
|
> {
|
|
1667
|
-
return
|
|
1732
|
+
return this.admitWrite((db) => purgeStoredEgressAuditReceipts(db));
|
|
1668
1733
|
}
|
|
1669
1734
|
|
|
1670
1735
|
getContextGeneration(): number {
|
|
@@ -2330,6 +2395,7 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
|
|
|
2330
2395
|
mirrorHashes: string[],
|
|
2331
2396
|
options: {
|
|
2332
2397
|
collection?: string;
|
|
2398
|
+
collections?: string[];
|
|
2333
2399
|
activeOnly?: boolean;
|
|
2334
2400
|
} = {}
|
|
2335
2401
|
): Promise<StoreResult<DocumentRow[]>> {
|
|
@@ -2350,7 +2416,7 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
|
|
|
2350
2416
|
|
|
2351
2417
|
// SQLite SQLITE_LIMIT_VARIABLE_NUMBER defaults to 999.
|
|
2352
2418
|
// Reserve headroom for optional non-IN parameters.
|
|
2353
|
-
const SQL_PARAM_LIMIT =
|
|
2419
|
+
const SQL_PARAM_LIMIT = 898;
|
|
2354
2420
|
|
|
2355
2421
|
for (let i = 0; i < uniqueHashes.length; i += SQL_PARAM_LIMIT) {
|
|
2356
2422
|
const batch = uniqueHashes.slice(i, i + SQL_PARAM_LIMIT);
|
|
@@ -2365,6 +2431,10 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
|
|
|
2365
2431
|
clauses.push("collection = ?");
|
|
2366
2432
|
params.push(options.collection);
|
|
2367
2433
|
}
|
|
2434
|
+
if (options.collections !== undefined) {
|
|
2435
|
+
clauses.push("collection IN (SELECT value FROM json_each(?))");
|
|
2436
|
+
params.push(JSON.stringify(options.collections));
|
|
2437
|
+
}
|
|
2368
2438
|
|
|
2369
2439
|
const sql = `SELECT * FROM documents INDEXED BY idx_documents_mirror_hash WHERE ${clauses.join(" AND ")} ORDER BY id`;
|
|
2370
2440
|
rows.push(...db.query<DbDocumentRow, string[]>(sql).all(...params));
|
|
@@ -90,6 +90,10 @@ export function buildEligibleDocumentQuery(
|
|
|
90
90
|
conditions.push("d.collection = ?");
|
|
91
91
|
params.push(options.collection);
|
|
92
92
|
}
|
|
93
|
+
if (options.collections !== undefined) {
|
|
94
|
+
conditions.push("d.collection IN (SELECT value FROM json_each(?))");
|
|
95
|
+
params.push(JSON.stringify(options.collections));
|
|
96
|
+
}
|
|
93
97
|
if (options.relPathPrefix !== undefined) {
|
|
94
98
|
conditions.push(
|
|
95
99
|
"(COALESCE(NULLIF(d.record_source_path, ''), d.rel_path) = ? OR substr(COALESCE(NULLIF(d.record_source_path, ''), d.rel_path), 1, length(?) + 1) = ? || '/')"
|
package/src/store/types.ts
CHANGED
|
@@ -718,6 +718,11 @@ export interface DocumentEligibilityOptions {
|
|
|
718
718
|
exclude?: string[];
|
|
719
719
|
/** Filter by collection */
|
|
720
720
|
collection?: string;
|
|
721
|
+
/**
|
|
722
|
+
* Filter to documents in ANY of these collections, before ranking and LIMIT.
|
|
723
|
+
* Undefined is unrestricted; an empty array denies all.
|
|
724
|
+
*/
|
|
725
|
+
collections?: string[];
|
|
721
726
|
/** Internal exact relative-path boundary applied before ranking and LIMIT. */
|
|
722
727
|
relPathPrefix?: string;
|
|
723
728
|
/** Filter to docs with ANY of these tags */
|
|
@@ -1945,6 +1950,8 @@ export interface StorePort {
|
|
|
1945
1950
|
mirrorHashes: string[],
|
|
1946
1951
|
options?: {
|
|
1947
1952
|
collection?: string;
|
|
1953
|
+
/** Keep only documents in ANY of these collections; empty denies all. */
|
|
1954
|
+
collections?: string[];
|
|
1948
1955
|
activeOnly?: boolean;
|
|
1949
1956
|
}
|
|
1950
1957
|
): Promise<StoreResult<DocumentRow[]>>;
|