@appsoftwareltd/etherpk-mcp 0.8.1 → 0.8.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/main.js +1830 -380
- package/dist/main.js.map +1 -1
- package/package.json +3 -3
package/dist/main.js
CHANGED
|
@@ -20,7 +20,7 @@ import * as Y from "yjs";
|
|
|
20
20
|
import { z } from "zod";
|
|
21
21
|
import { Awareness, applyAwarenessUpdate, encodeAwarenessUpdate, removeAwarenessStates } from "y-protocols/awareness";
|
|
22
22
|
import * as encoding from "lib0/encoding";
|
|
23
|
-
import { parse,
|
|
23
|
+
import { Document, Scalar, YAMLMap, isMap, isScalar, isSeq, parse, parseDocument, visit } from "yaml";
|
|
24
24
|
import "fake-indexeddb/auto";
|
|
25
25
|
import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
26
26
|
import { classHighlighter, highlightTree } from "@lezer/highlight";
|
|
@@ -34,7 +34,7 @@ import { request } from "node:https";
|
|
|
34
34
|
import { BlockList, isIP } from "node:net";
|
|
35
35
|
var package_default = {
|
|
36
36
|
name: "@appsoftwareltd/etherpk-mcp",
|
|
37
|
-
version: "0.8.
|
|
37
|
+
version: "0.8.3",
|
|
38
38
|
license: "Elastic-2.0",
|
|
39
39
|
description: "EtherPK Headless Client: an MCP server over a synced knowledge graph, run beside the agent on the user's own machine.",
|
|
40
40
|
type: "module",
|
|
@@ -635,7 +635,9 @@ function createSyncApi(deps) {
|
|
|
635
635
|
});
|
|
636
636
|
if (!res.ok) {
|
|
637
637
|
const body = await res.json().catch(() => null);
|
|
638
|
-
|
|
638
|
+
const nested = typeof body?.error === "object" && body.error !== null ? body.error : null;
|
|
639
|
+
const code = nested?.code ?? body?.code;
|
|
640
|
+
throw new SyncApiError(nested?.message ?? body?.code ?? `HTTP ${res.status}`, res.status, code, body?.retryable === true);
|
|
639
641
|
}
|
|
640
642
|
return await res.json();
|
|
641
643
|
}
|
|
@@ -674,8 +676,14 @@ function createSyncApi(deps) {
|
|
|
674
676
|
method: "PUT",
|
|
675
677
|
body: JSON.stringify({ publicKey })
|
|
676
678
|
}),
|
|
677
|
-
getIdentityByEmail: (email) => call(
|
|
678
|
-
|
|
679
|
+
getIdentityByEmail: (graphId, email) => call("/api/v1/sync/identity/lookup", {
|
|
680
|
+
method: "POST",
|
|
681
|
+
body: JSON.stringify({
|
|
682
|
+
graphId,
|
|
683
|
+
email
|
|
684
|
+
})
|
|
685
|
+
}).catch((e) => {
|
|
686
|
+
if (e instanceof SyncApiError && e.status === 404) return null;
|
|
679
687
|
throw e;
|
|
680
688
|
}),
|
|
681
689
|
createInvite: (graphId, inviteeEmail, sealedKeyring) => call("/api/v1/sync/invites", {
|
|
@@ -688,6 +696,7 @@ function createSyncApi(deps) {
|
|
|
688
696
|
}),
|
|
689
697
|
listInvites: () => call("/api/v1/sync/invites").then((r) => r.invites),
|
|
690
698
|
acceptInvite: (id) => call(`/api/v1/sync/invites/${id}/accept`, { method: "POST" }).then((r) => r.graphId),
|
|
699
|
+
withdrawInvite: (id) => call(`/api/v1/sync/invites/${id}`, { method: "DELETE" }),
|
|
691
700
|
graphMembers: (graphId) => call(`/api/v1/sync/graphs/${graphId}/members`).then((r) => r.members),
|
|
692
701
|
leaveGraph: (graphId) => call(`/api/v1/sync/graphs/${graphId}/leave`, { method: "POST" }),
|
|
693
702
|
deleteGraph: (graphId) => call(`/api/v1/sync/graphs/${graphId}`, { method: "DELETE" }),
|
|
@@ -785,6 +794,36 @@ function createSyncTokenSource(mint, opts) {
|
|
|
785
794
|
};
|
|
786
795
|
}
|
|
787
796
|
//#endregion
|
|
797
|
+
//#region src/connection-errors.ts
|
|
798
|
+
/** Socket and TLS error codes Node's fetch reports as the cause of "fetch failed". */
|
|
799
|
+
var TLS_CODES = new Set([
|
|
800
|
+
"ERR_SSL_WRONG_VERSION_NUMBER",
|
|
801
|
+
"EPROTO",
|
|
802
|
+
"CERT_HAS_EXPIRED",
|
|
803
|
+
"DEPTH_ZERO_SELF_SIGNED_CERT",
|
|
804
|
+
"SELF_SIGNED_CERT_IN_CHAIN",
|
|
805
|
+
"UNABLE_TO_VERIFY_LEAF_SIGNATURE",
|
|
806
|
+
"ERR_TLS_CERT_ALTNAME_INVALID"
|
|
807
|
+
]);
|
|
808
|
+
/**
|
|
809
|
+
* Why connecting to a Sync Server failed, in terms a person can act on, or null for a failure
|
|
810
|
+
* this does not recognise. The first mistakes a login makes - the EtherPK app's address in place
|
|
811
|
+
* of the server's, a wrong or revoked token, https against a plain-HTTP server, a mistyped host -
|
|
812
|
+
* otherwise read as "HTTP 404", "Unauthorized" or "fetch failed".
|
|
813
|
+
*/
|
|
814
|
+
function describeConnectionFailure(error, syncServer) {
|
|
815
|
+
if (error instanceof SyncApiError) {
|
|
816
|
+
if (error.status === 404 && error.message === "HTTP 404") return `${syncServer} is not a Sync Server (it may be the EtherPK app). Use the Sync Server address shown in EtherPK under a synced graph's Settings > Agents.`;
|
|
817
|
+
if (error.status === 401) return `${syncServer} did not accept the access token: it may be revoked, expired or mistyped. Create one at ${syncServer}/account/tokens and run login again.`;
|
|
818
|
+
return null;
|
|
819
|
+
}
|
|
820
|
+
if (error instanceof TypeError && error.message === "fetch failed") {
|
|
821
|
+
const code = error.cause?.code;
|
|
822
|
+
return `Could not connect to ${syncServer}: ${code === "ENOTFOUND" ? "the host was not found. Check the address." : code === "ECONNREFUSED" ? "the connection was refused. Check the address and that the server is running." : typeof code === "string" && TLS_CODES.has(code) ? "the secure connection failed. Check whether the server uses https or http." : `it did not answer (${typeof code === "string" ? code : "no reason given"}).`}`;
|
|
823
|
+
}
|
|
824
|
+
return null;
|
|
825
|
+
}
|
|
826
|
+
//#endregion
|
|
788
827
|
//#region src/account.ts
|
|
789
828
|
/**
|
|
790
829
|
* The account side of the [[Headless Client]]: the same REST bridge and vault the browser uses,
|
|
@@ -806,7 +845,13 @@ function createHeadlessAccount(config) {
|
|
|
806
845
|
/** Resolve the PAT to its principal; a graph-scoped or revoked token fails here, in words. */
|
|
807
846
|
async function connectAccount(config) {
|
|
808
847
|
const account = createHeadlessAccount(config);
|
|
809
|
-
|
|
848
|
+
let me;
|
|
849
|
+
try {
|
|
850
|
+
me = await account.api.me();
|
|
851
|
+
} catch (error) {
|
|
852
|
+
const described = describeConnectionFailure(error, config.syncServer);
|
|
853
|
+
throw described ? new Error(described, { cause: error }) : error;
|
|
854
|
+
}
|
|
810
855
|
return {
|
|
811
856
|
...account,
|
|
812
857
|
principal: {
|
|
@@ -2466,6 +2511,119 @@ function snippetSegments(snippet) {
|
|
|
2466
2511
|
}
|
|
2467
2512
|
return segments.filter((s) => s.text !== "");
|
|
2468
2513
|
}
|
|
2514
|
+
/** Letters, digits, `_`, `.` and `-`: a key a person can type, dot paths included. */
|
|
2515
|
+
var PROPERTY_KEY_PATTERN = /^[\p{L}\p{N}_][\p{L}\p{N}_.-]*$/u;
|
|
2516
|
+
/**
|
|
2517
|
+
* Whitespace-separated tokens. A quote opens a run that spaces do not end, whether it starts the
|
|
2518
|
+
* token (`"a phrase"`) or follows a key (`status:"in progress"`); an unclosed one runs to the end.
|
|
2519
|
+
*/
|
|
2520
|
+
function searchTokens(input) {
|
|
2521
|
+
const out = [];
|
|
2522
|
+
let index = 0;
|
|
2523
|
+
while (index < input.length) {
|
|
2524
|
+
if (/\s/.test(input[index])) {
|
|
2525
|
+
index++;
|
|
2526
|
+
continue;
|
|
2527
|
+
}
|
|
2528
|
+
const from = index;
|
|
2529
|
+
while (index < input.length && !/\s/.test(input[index])) {
|
|
2530
|
+
if (input[index] === "\"") {
|
|
2531
|
+
const close = input.indexOf("\"", index + 1);
|
|
2532
|
+
index = close === -1 ? input.length : close + 1;
|
|
2533
|
+
continue;
|
|
2534
|
+
}
|
|
2535
|
+
index++;
|
|
2536
|
+
}
|
|
2537
|
+
out.push({
|
|
2538
|
+
text: input.slice(from, index),
|
|
2539
|
+
from,
|
|
2540
|
+
to: index
|
|
2541
|
+
});
|
|
2542
|
+
}
|
|
2543
|
+
return out;
|
|
2544
|
+
}
|
|
2545
|
+
/**
|
|
2546
|
+
* The filters in `input`, and the words left for name and text matching.
|
|
2547
|
+
*
|
|
2548
|
+
* A term is a filter only when its key is one at least one searchable document carries
|
|
2549
|
+
* (`knownKeys`, lower-cased): anything else shaped like `x:y` stays words, which keeps a URL, a
|
|
2550
|
+
* time or `note:` in prose as text. A fully quoted term is always words. A known key with no
|
|
2551
|
+
* value yet (`status:` mid-typing) is dropped from both, so the key is never searched as a word
|
|
2552
|
+
* a moment before the value arrives.
|
|
2553
|
+
*/
|
|
2554
|
+
function parseSearchQuery(input, knownKeys) {
|
|
2555
|
+
const filters = [];
|
|
2556
|
+
const terms = [];
|
|
2557
|
+
const cut = [];
|
|
2558
|
+
if (knownKeys.size > 0) for (const token of searchTokens(input)) {
|
|
2559
|
+
if (token.text.startsWith("\"")) continue;
|
|
2560
|
+
const colon = token.text.indexOf(":");
|
|
2561
|
+
if (colon <= 0) continue;
|
|
2562
|
+
const negated = token.text.startsWith("-");
|
|
2563
|
+
const key = token.text.slice(negated ? 1 : 0, colon);
|
|
2564
|
+
if (!PROPERTY_KEY_PATTERN.test(key) || !knownKeys.has(key.toLowerCase())) continue;
|
|
2565
|
+
const filter = filterValue(key, token.text.slice(colon + 1), negated);
|
|
2566
|
+
cut.push({
|
|
2567
|
+
from: token.from,
|
|
2568
|
+
to: token.to
|
|
2569
|
+
});
|
|
2570
|
+
if (filter === null) continue;
|
|
2571
|
+
filters.push(filter);
|
|
2572
|
+
terms.push({
|
|
2573
|
+
filter,
|
|
2574
|
+
from: token.from,
|
|
2575
|
+
to: token.to
|
|
2576
|
+
});
|
|
2577
|
+
}
|
|
2578
|
+
return {
|
|
2579
|
+
words: withoutRanges(input, cut),
|
|
2580
|
+
filters,
|
|
2581
|
+
terms
|
|
2582
|
+
};
|
|
2583
|
+
}
|
|
2584
|
+
/** The filter for a value as typed, or null when there is no value yet. */
|
|
2585
|
+
function filterValue(key, raw, negated) {
|
|
2586
|
+
if (raw === "*") return {
|
|
2587
|
+
key,
|
|
2588
|
+
value: null,
|
|
2589
|
+
prefix: false,
|
|
2590
|
+
negated
|
|
2591
|
+
};
|
|
2592
|
+
if (raw.startsWith("\"")) {
|
|
2593
|
+
const close = raw.indexOf("\"", 1);
|
|
2594
|
+
const value = (close === -1 ? raw.slice(1) : raw.slice(1, close)).trim();
|
|
2595
|
+
return value === "" ? null : {
|
|
2596
|
+
key,
|
|
2597
|
+
value,
|
|
2598
|
+
prefix: false,
|
|
2599
|
+
negated
|
|
2600
|
+
};
|
|
2601
|
+
}
|
|
2602
|
+
if (raw.length > 1 && raw.endsWith("*")) return {
|
|
2603
|
+
key,
|
|
2604
|
+
value: raw.slice(0, -1),
|
|
2605
|
+
prefix: true,
|
|
2606
|
+
negated
|
|
2607
|
+
};
|
|
2608
|
+
return raw === "" ? null : {
|
|
2609
|
+
key,
|
|
2610
|
+
value: raw,
|
|
2611
|
+
prefix: false,
|
|
2612
|
+
negated
|
|
2613
|
+
};
|
|
2614
|
+
}
|
|
2615
|
+
/** `input` with the ranges removed and the gaps they leave closed to single spaces. */
|
|
2616
|
+
function withoutRanges(input, ranges) {
|
|
2617
|
+
if (ranges.length === 0) return input;
|
|
2618
|
+
const parts = [];
|
|
2619
|
+
let at = 0;
|
|
2620
|
+
for (const range of ranges) {
|
|
2621
|
+
parts.push(input.slice(at, range.from));
|
|
2622
|
+
at = range.to;
|
|
2623
|
+
}
|
|
2624
|
+
parts.push(input.slice(at));
|
|
2625
|
+
return parts.map((part) => part.trim()).filter((part) => part !== "").join(" ");
|
|
2626
|
+
}
|
|
2469
2627
|
//#endregion
|
|
2470
2628
|
//#region ../client/src/lib/document/index-db.ts
|
|
2471
2629
|
/**
|
|
@@ -2527,6 +2685,15 @@ CREATE TABLE IF NOT EXISTS passages (
|
|
|
2527
2685
|
page_id INTEGER NOT NULL, ord INTEGER NOT NULL, start_line INTEGER NOT NULL,
|
|
2528
2686
|
end_line INTEGER NOT NULL, first_block_local_id INTEGER NOT NULL,
|
|
2529
2687
|
content_hash TEXT NOT NULL, text TEXT NOT NULL);
|
|
2688
|
+
-- [[Property]] rows for [[Property Filter]]s (ADR 0107): one per value, keys as dot paths. key and
|
|
2689
|
+
-- value keep their spelling for display; key_lc and value_lc are lower-cased in JS, because
|
|
2690
|
+
-- SQLite's lower() folds ASCII only and a filter matches ignoring case in any script. value is
|
|
2691
|
+
-- NULL on a mapping's presence row. title and aliases rows come from the graph's names. A
|
|
2692
|
+
-- protected document has none.
|
|
2693
|
+
CREATE TABLE IF NOT EXISTS properties (
|
|
2694
|
+
page_id INTEGER NOT NULL, key TEXT NOT NULL, key_lc TEXT NOT NULL,
|
|
2695
|
+
value TEXT, value_lc TEXT);
|
|
2696
|
+
CREATE INDEX IF NOT EXISTS properties_key ON properties(key_lc, value_lc);
|
|
2530
2697
|
CREATE INDEX IF NOT EXISTS passages_hash ON passages(content_hash);
|
|
2531
2698
|
CREATE INDEX IF NOT EXISTS links_concept_key ON links(concept_key);
|
|
2532
2699
|
CREATE INDEX IF NOT EXISTS blocks_page ON blocks(page_id);
|
|
@@ -2575,7 +2742,7 @@ function createSchema(db) {
|
|
|
2575
2742
|
db.exec(SCHEMA$1);
|
|
2576
2743
|
db.run("INSERT OR IGNORE INTO index_metadata (key, value) VALUES ('active_generation', 1)");
|
|
2577
2744
|
db.run("INSERT OR IGNORE INTO index_metadata (key, value) VALUES ('revision', 0)");
|
|
2578
|
-
db.exec(`PRAGMA user_version =
|
|
2745
|
+
db.exec(`PRAGMA user_version = 14`);
|
|
2579
2746
|
}
|
|
2580
2747
|
function activeIndexGeneration(db) {
|
|
2581
2748
|
return db.all("SELECT value FROM index_metadata WHERE key = 'active_generation'")[0]?.value ?? 1;
|
|
@@ -2596,7 +2763,7 @@ function advanceIndexRevision(db) {
|
|
|
2596
2763
|
*/
|
|
2597
2764
|
function isUsableIndex(db) {
|
|
2598
2765
|
try {
|
|
2599
|
-
return db.all("PRAGMA user_version")[0]?.user_version ===
|
|
2766
|
+
return db.all("PRAGMA user_version")[0]?.user_version === 14;
|
|
2600
2767
|
} catch {
|
|
2601
2768
|
return false;
|
|
2602
2769
|
}
|
|
@@ -2616,8 +2783,11 @@ function documentHash(db, key) {
|
|
|
2616
2783
|
*/
|
|
2617
2784
|
function indexDocHash(doc) {
|
|
2618
2785
|
const includes = doc.includes ?? [];
|
|
2619
|
-
|
|
2620
|
-
|
|
2786
|
+
const properties = doc.properties ?? [];
|
|
2787
|
+
if (includes.length === 0 && properties.length === 0) return hashText(doc.text);
|
|
2788
|
+
const includeText = includes.map((i) => `${i.publication}\u0001${i.slot}\u0001${i.concept}`).join("\0");
|
|
2789
|
+
const propertyText = properties.map((p) => `${p.key}\u0001${p.value ?? ""}`).join("\0");
|
|
2790
|
+
return hashText(`${doc.text}\u0000${includeText}\u0003${propertyText}`);
|
|
2621
2791
|
}
|
|
2622
2792
|
function hashText(text) {
|
|
2623
2793
|
let h1 = 3735928559;
|
|
@@ -2647,13 +2817,18 @@ function hashText(text) {
|
|
|
2647
2817
|
function isTextSearchable(block) {
|
|
2648
2818
|
return !containsCipherFence(block.text);
|
|
2649
2819
|
}
|
|
2650
|
-
/**
|
|
2820
|
+
/**
|
|
2821
|
+
* Write one document's derived rows under an already-established page id. Returns the text of
|
|
2822
|
+
* every row it put into the text index, which `ingestOne` compares with what it replaced.
|
|
2823
|
+
*/
|
|
2651
2824
|
function insertDerived(db, pageId, doc) {
|
|
2825
|
+
const searchable = [];
|
|
2652
2826
|
for (const alias of doc.aliases) db.run("INSERT INTO aliases (page_id, alias_key, display) VALUES (?,?,?)", [
|
|
2653
2827
|
pageId,
|
|
2654
2828
|
conceptKey$1(alias),
|
|
2655
2829
|
alias
|
|
2656
2830
|
]);
|
|
2831
|
+
insertProperties(db, pageId, doc);
|
|
2657
2832
|
for (const include of doc.includes ?? []) db.run("INSERT INTO publication_includes (page_id, publication_id, slot, concept_key, concept) VALUES (?,?,?,?,?)", [
|
|
2658
2833
|
pageId,
|
|
2659
2834
|
include.publication,
|
|
@@ -2686,7 +2861,10 @@ function insertDerived(db, pageId, doc) {
|
|
|
2686
2861
|
b.label,
|
|
2687
2862
|
b.text
|
|
2688
2863
|
]);
|
|
2689
|
-
if (isTextSearchable(b) && b.localId < 1048576)
|
|
2864
|
+
if (isTextSearchable(b) && b.localId < 1048576) {
|
|
2865
|
+
db.run("INSERT INTO block_fts (rowid, text) VALUES (?,?)", [pageId * BLOCK_FTS_STRIDE + b.localId, b.text]);
|
|
2866
|
+
searchable.push(b.text);
|
|
2867
|
+
}
|
|
2690
2868
|
}
|
|
2691
2869
|
for (const l of links) db.run("INSERT INTO links (page_id, concept, concept_key, line, line_text, match_start, match_end, block_local_id) VALUES (?,?,?,?,?,?,?,?)", [
|
|
2692
2870
|
pageId,
|
|
@@ -2724,6 +2902,35 @@ function insertDerived(db, pageId, doc) {
|
|
|
2724
2902
|
hashText(passage.text),
|
|
2725
2903
|
passage.text
|
|
2726
2904
|
]);
|
|
2905
|
+
return searchable;
|
|
2906
|
+
}
|
|
2907
|
+
/**
|
|
2908
|
+
* Write one document's [[Property]] rows (ADR 0107): its name as `title`, each alias as
|
|
2909
|
+
* `aliases`, then what its Frontmatter carries. A [[Protected Document]] writes none. Its
|
|
2910
|
+
* Frontmatter is stored in plaintext, but no kind of matching in Search reaches protected
|
|
2911
|
+
* content, and a document with no rows cannot match a filter, not even a negated one, because
|
|
2912
|
+
* every filtered query also requires `protected = 0`.
|
|
2913
|
+
*/
|
|
2914
|
+
function insertProperties(db, pageId, doc) {
|
|
2915
|
+
if (protectedFlag(doc) === 1) return;
|
|
2916
|
+
const rows = [
|
|
2917
|
+
{
|
|
2918
|
+
key: "title",
|
|
2919
|
+
value: doc.concept
|
|
2920
|
+
},
|
|
2921
|
+
...doc.aliases.map((alias) => ({
|
|
2922
|
+
key: "aliases",
|
|
2923
|
+
value: alias
|
|
2924
|
+
})),
|
|
2925
|
+
...doc.properties ?? []
|
|
2926
|
+
];
|
|
2927
|
+
for (const row of rows) db.run("INSERT INTO properties (page_id, key, key_lc, value, value_lc) VALUES (?,?,?,?,?)", [
|
|
2928
|
+
pageId,
|
|
2929
|
+
row.key,
|
|
2930
|
+
row.key.toLowerCase(),
|
|
2931
|
+
row.value,
|
|
2932
|
+
row.value === null ? null : row.value.toLowerCase()
|
|
2933
|
+
]);
|
|
2727
2934
|
}
|
|
2728
2935
|
/**
|
|
2729
2936
|
* Write one document's [[Task Concept]] rows (ADR 0051).
|
|
@@ -2731,7 +2938,7 @@ function insertDerived(db, pageId, doc) {
|
|
|
2731
2938
|
* Derivation supplies the wikilinked half; the document's own concept is added HERE, as the
|
|
2732
2939
|
* virtual root of every task's ancestry chain, because derivation reads text and never learns
|
|
2733
2940
|
* the document's identity. Without that row, filtering to a project page returns nothing for
|
|
2734
|
-
* the tasks written directly on it — the case the "
|
|
2941
|
+
* the tasks written directly on it — the case the "Filter to <active tab>" shortcut exists for.
|
|
2735
2942
|
*
|
|
2736
2943
|
* Only the document's own concept goes in, never its aliases: the query resolves a filtered
|
|
2737
2944
|
* name to its page and matches on every name that page answers to, so an alias row here would
|
|
@@ -2768,6 +2975,37 @@ function inTransaction(db, work) {
|
|
|
2768
2975
|
throw err;
|
|
2769
2976
|
}
|
|
2770
2977
|
}
|
|
2978
|
+
/**
|
|
2979
|
+
* Merge the text index into one segment, which is what removes the words of deleted rows from
|
|
2980
|
+
* the database file (ADR 0097).
|
|
2981
|
+
*
|
|
2982
|
+
* FTS5 deletes by writing a tombstone: the deleted row's terms stay in the segment that holds
|
|
2983
|
+
* them until a merge folds segment and tombstone together, and in a graph nobody is editing that
|
|
2984
|
+
* can be never. `PRAGMA secure_delete` (index-db-sqlite.ts) zeroes the pages a merge frees, but a
|
|
2985
|
+
* segment still in use is not freed. Without a merge, a document protected with content would
|
|
2986
|
+
* leave its words readable with `strings` in the browser's OPFS pool and the Headless Client's
|
|
2987
|
+
* index file, which is the disk-level reader protection exists to stop.
|
|
2988
|
+
*
|
|
2989
|
+
* FTS5's own `secure-delete` option would remove the terms on every delete instead. Measured on a
|
|
2990
|
+
* 2,400-document index it made an edit's re-index 3.5x slower and a generation swap 12.5 s rather
|
|
2991
|
+
* than 0.6 s, for every graph. `optimize` costs about 70 ms there and runs only when protected
|
|
2992
|
+
* text may have been left behind: see {@link ingestOne} and {@link purgeIfProtected}.
|
|
2993
|
+
*
|
|
2994
|
+
* "Protected" here is the index's own flag, `containsCipherFence` anywhere in the text, so it
|
|
2995
|
+
* also covers a page holding a fence beside plaintext (text typed after a protected fence, or a
|
|
2996
|
+
* quoted example), whose plaintext is indexed until the page is protected whole.
|
|
2997
|
+
*/
|
|
2998
|
+
function purgeDeletedText(db) {
|
|
2999
|
+
db.run("INSERT INTO block_fts(block_fts) VALUES('optimize')");
|
|
3000
|
+
}
|
|
3001
|
+
/**
|
|
3002
|
+
* After a rebuild: purge when the index now holds any protected document. A rebuild replaces
|
|
3003
|
+
* every row, so the old segments may hold the plaintext of a document that was protected since
|
|
3004
|
+
* the last build, on this device or another. A graph with no protected document pays nothing.
|
|
3005
|
+
*/
|
|
3006
|
+
function purgeIfProtected(db, generation) {
|
|
3007
|
+
if (db.all("SELECT 1 AS n FROM pages WHERE generation = ? AND protected = 1 LIMIT 1", [generation]).length > 0) purgeDeletedText(db);
|
|
3008
|
+
}
|
|
2771
3009
|
/** SQLite has no boolean: the `protected` column as stored. */
|
|
2772
3010
|
function protectedFlag(doc) {
|
|
2773
3011
|
return containsCipherFence(doc.text) ? 1 : 0;
|
|
@@ -2795,7 +3033,8 @@ function deleteGeneration(db, generation) {
|
|
|
2795
3033
|
"links",
|
|
2796
3034
|
"tasks",
|
|
2797
3035
|
"task_concepts",
|
|
2798
|
-
"passages"
|
|
3036
|
+
"passages",
|
|
3037
|
+
"properties"
|
|
2799
3038
|
]) db.run(`DELETE FROM ${table} WHERE page_id IN (SELECT id FROM pages WHERE generation = ?)`, [generation]);
|
|
2800
3039
|
db.run("DELETE FROM pages WHERE generation = ?", [generation]);
|
|
2801
3040
|
}
|
|
@@ -2822,6 +3061,7 @@ function commitIndexRebuild(db, generation) {
|
|
|
2822
3061
|
const revision = advanceIndexRevision(db);
|
|
2823
3062
|
const obsolete = db.all("SELECT DISTINCT generation FROM pages WHERE generation <> ?", [generation]).map((row) => row.generation);
|
|
2824
3063
|
for (const oldGeneration of obsolete) deleteGeneration(db, oldGeneration);
|
|
3064
|
+
purgeIfProtected(db, generation);
|
|
2825
3065
|
return revision;
|
|
2826
3066
|
});
|
|
2827
3067
|
}
|
|
@@ -2857,6 +3097,7 @@ function ingestOne(db, doc) {
|
|
|
2857
3097
|
const generation = activeIndexGeneration(db);
|
|
2858
3098
|
const found = db.all("SELECT id FROM pages WHERE generation = ? AND concept_key = ?", [generation, key]);
|
|
2859
3099
|
let pageId;
|
|
3100
|
+
const leaving = found.length > 0 && protectedFlag(doc) === 1 ? db.all("SELECT text FROM block_fts WHERE rowid >= ? AND rowid < ?", blockFtsRange(found[0].id)).map((row) => row.text) : [];
|
|
2860
3101
|
if (found.length > 0) {
|
|
2861
3102
|
pageId = found[0].id;
|
|
2862
3103
|
db.run("DELETE FROM aliases WHERE page_id = ?", [pageId]);
|
|
@@ -2868,6 +3109,7 @@ function ingestOne(db, doc) {
|
|
|
2868
3109
|
db.run("DELETE FROM tasks WHERE page_id = ?", [pageId]);
|
|
2869
3110
|
db.run("DELETE FROM task_concepts WHERE page_id = ?", [pageId]);
|
|
2870
3111
|
db.run("DELETE FROM passages WHERE page_id = ?", [pageId]);
|
|
3112
|
+
db.run("DELETE FROM properties WHERE page_id = ?", [pageId]);
|
|
2871
3113
|
db.run("UPDATE pages SET concept = ?, kind = ?, text_hash = ?, protected = ? WHERE id = ?", [
|
|
2872
3114
|
doc.concept,
|
|
2873
3115
|
doc.kind,
|
|
@@ -2887,7 +3129,8 @@ function ingestOne(db, doc) {
|
|
|
2887
3129
|
protectedFlag(doc)
|
|
2888
3130
|
]);
|
|
2889
3131
|
}
|
|
2890
|
-
insertDerived(db, pageId, doc);
|
|
3132
|
+
const written = new Set(insertDerived(db, pageId, doc));
|
|
3133
|
+
if (leaving.some((text) => !written.has(text))) purgeDeletedText(db);
|
|
2891
3134
|
});
|
|
2892
3135
|
}
|
|
2893
3136
|
/** Every concept key that resolves to a real document (canonical name or alias). */
|
|
@@ -3218,7 +3461,8 @@ function backlinksFor(db, concept) {
|
|
|
3218
3461
|
const hits = db.all(`SELECT l.page_id, p.concept AS sourceConcept, p.kind AS sourceKind, l.line, l.line_text,
|
|
3219
3462
|
l.match_start, l.match_end, l.block_local_id, l.in_title
|
|
3220
3463
|
FROM links l JOIN pages p ON p.id = l.page_id
|
|
3221
|
-
WHERE p.generation=? AND l.concept_key IN (${placeholders})
|
|
3464
|
+
WHERE p.generation=? AND l.concept_key IN (${placeholders})
|
|
3465
|
+
ORDER BY l.page_id, l.line, l.match_start`, [generation, ...names]);
|
|
3222
3466
|
const blocksByPage = /* @__PURE__ */ new Map();
|
|
3223
3467
|
for (const pid of new Set(hits.map((h) => h.page_id))) {
|
|
3224
3468
|
const rows = db.all("SELECT local_id, parent_local_id, ord, kind, depth, done, label, text FROM blocks WHERE page_id=? ORDER BY local_id", [pid]);
|
|
@@ -3238,7 +3482,11 @@ function backlinksFor(db, concept) {
|
|
|
3238
3482
|
blocksByPage.set(pid, blocks);
|
|
3239
3483
|
}
|
|
3240
3484
|
const groups = /* @__PURE__ */ new Map();
|
|
3485
|
+
const places = /* @__PURE__ */ new Set();
|
|
3241
3486
|
for (const h of hits) {
|
|
3487
|
+
const place = h.in_title ? `${h.page_id}:title` : h.block_local_id != null ? `${h.page_id}:block:${h.block_local_id}` : `${h.page_id}:line:${h.line}`;
|
|
3488
|
+
if (places.has(place)) continue;
|
|
3489
|
+
places.add(place);
|
|
3242
3490
|
let group = groups.get(h.sourceConcept);
|
|
3243
3491
|
if (!group) {
|
|
3244
3492
|
group = {
|
|
@@ -3274,25 +3522,27 @@ var SNIPPET_TOKENS = 24;
|
|
|
3274
3522
|
* scores merely reintroduces the long-document bias bm25 exists to remove. Counting is cruder,
|
|
3275
3523
|
* robust, and explainable in one sentence - and it orders by a number the row displays.
|
|
3276
3524
|
*/
|
|
3277
|
-
function searchText$1(db, query, offset, limit) {
|
|
3525
|
+
function searchText$1(db, query, offset, limit, filters = []) {
|
|
3278
3526
|
const match = buildFtsMatch(query);
|
|
3279
3527
|
if (match === null) return {
|
|
3280
3528
|
groups: [],
|
|
3281
3529
|
hasMore: false
|
|
3282
3530
|
};
|
|
3283
3531
|
const generation = activeIndexGeneration(db);
|
|
3532
|
+
const narrow = propertyFilterSql(filters);
|
|
3284
3533
|
const ranked = db.all(`WITH hits AS MATERIALIZED (
|
|
3285
3534
|
SELECT rowid / ${BLOCK_FTS_STRIDE} AS page_id, bm25(block_fts) AS score
|
|
3286
3535
|
FROM block_fts WHERE block_fts MATCH ?
|
|
3287
3536
|
)
|
|
3288
3537
|
SELECT h.page_id AS page_id, COUNT(*) AS matches, MIN(h.score) AS best
|
|
3289
3538
|
FROM hits h
|
|
3290
|
-
JOIN pages p ON p.id = h.page_id AND p.generation =
|
|
3539
|
+
JOIN pages p ON p.id = h.page_id AND p.generation = ?${narrow.sql}
|
|
3291
3540
|
GROUP BY h.page_id
|
|
3292
3541
|
ORDER BY matches DESC, best ASC, h.page_id ASC
|
|
3293
3542
|
LIMIT ? OFFSET ?`, [
|
|
3294
3543
|
match,
|
|
3295
3544
|
generation,
|
|
3545
|
+
...narrow.params,
|
|
3296
3546
|
limit + 1,
|
|
3297
3547
|
offset
|
|
3298
3548
|
]);
|
|
@@ -3370,21 +3620,23 @@ function searchBreadcrumb(blocks, localId) {
|
|
|
3370
3620
|
* Capped rather than exact because a prefix query - which is every query, mid-typing - can
|
|
3371
3621
|
* match most of the graph, and this runs beside the rows it must never delay.
|
|
3372
3622
|
*/
|
|
3373
|
-
function searchTextCount(db, query) {
|
|
3623
|
+
function searchTextCount(db, query, filters = []) {
|
|
3374
3624
|
const match = buildFtsMatch(query);
|
|
3375
3625
|
if (match === null) return {
|
|
3376
3626
|
total: 0,
|
|
3377
3627
|
capped: false
|
|
3378
3628
|
};
|
|
3629
|
+
const narrow = propertyFilterSql(filters);
|
|
3379
3630
|
const n = db.all(`SELECT COUNT(*) AS n FROM (
|
|
3380
3631
|
SELECT m.page_id FROM (
|
|
3381
3632
|
SELECT DISTINCT rowid / 1048576 AS page_id FROM block_fts
|
|
3382
3633
|
WHERE block_fts MATCH ?
|
|
3383
3634
|
) m
|
|
3384
|
-
JOIN pages p ON p.id = m.page_id AND p.generation =
|
|
3635
|
+
JOIN pages p ON p.id = m.page_id AND p.generation = ?${narrow.sql}
|
|
3385
3636
|
LIMIT ?)`, [
|
|
3386
3637
|
match,
|
|
3387
3638
|
activeIndexGeneration(db),
|
|
3639
|
+
...narrow.params,
|
|
3388
3640
|
1001
|
|
3389
3641
|
])[0]?.n ?? 0;
|
|
3390
3642
|
return n > 1e3 ? {
|
|
@@ -3395,6 +3647,106 @@ function searchTextCount(db, query) {
|
|
|
3395
3647
|
capped: false
|
|
3396
3648
|
};
|
|
3397
3649
|
}
|
|
3650
|
+
/** Documents a filtered name listing returns at most: well past any graph Search serves. */
|
|
3651
|
+
var PROPERTY_MATCH_CAP = 5e3;
|
|
3652
|
+
/**
|
|
3653
|
+
* The SQL that narrows a query over `pages AS p` to the documents passing every filter, as an
|
|
3654
|
+
* `AND …` suffix, with its parameters in order. Empty when there are no filters. A filtered
|
|
3655
|
+
* query never returns a [[Protected Document]], which has no property rows; the explicit
|
|
3656
|
+
* `protected = 0` is what keeps a NEGATED filter from matching one.
|
|
3657
|
+
*/
|
|
3658
|
+
function propertyFilterSql(filters) {
|
|
3659
|
+
if (filters.length === 0) return {
|
|
3660
|
+
sql: "",
|
|
3661
|
+
params: []
|
|
3662
|
+
};
|
|
3663
|
+
const clauses = ["p.protected = 0"];
|
|
3664
|
+
const params = [];
|
|
3665
|
+
for (const filter of filters) {
|
|
3666
|
+
let test = "x.key_lc = ?";
|
|
3667
|
+
params.push(filter.key.toLowerCase());
|
|
3668
|
+
if (filter.value !== null) {
|
|
3669
|
+
const value = filter.value.toLowerCase();
|
|
3670
|
+
if (filter.prefix) {
|
|
3671
|
+
test += " AND substr(x.value_lc, 1, ?) = ?";
|
|
3672
|
+
params.push(Array.from(value).length, value);
|
|
3673
|
+
} else {
|
|
3674
|
+
test += " AND x.value_lc = ?";
|
|
3675
|
+
params.push(value);
|
|
3676
|
+
}
|
|
3677
|
+
}
|
|
3678
|
+
clauses.push(`${filter.negated ? "NOT " : ""}EXISTS (SELECT 1 FROM properties x WHERE x.page_id = p.id AND ${test})`);
|
|
3679
|
+
}
|
|
3680
|
+
return {
|
|
3681
|
+
sql: ` AND ${clauses.join(" AND ")}`,
|
|
3682
|
+
params
|
|
3683
|
+
};
|
|
3684
|
+
}
|
|
3685
|
+
/**
|
|
3686
|
+
* Every key a searchable document carries, once per spelling, most used first: what decides
|
|
3687
|
+
* whether a typed `x:y` is a filter, and what the key suggestions offer.
|
|
3688
|
+
*/
|
|
3689
|
+
function propertyKeys(db) {
|
|
3690
|
+
return db.all(`SELECT pr.key AS key, COUNT(DISTINCT pr.page_id) AS documents
|
|
3691
|
+
FROM properties pr JOIN pages p ON p.id = pr.page_id
|
|
3692
|
+
WHERE p.generation = ?
|
|
3693
|
+
GROUP BY pr.key
|
|
3694
|
+
ORDER BY documents DESC, pr.key_lc, pr.key`, [activeIndexGeneration(db)]);
|
|
3695
|
+
}
|
|
3696
|
+
/** The values `key` has across the graph, grouped ignoring case, most used first. */
|
|
3697
|
+
function propertyValues(db, key, limit = 50) {
|
|
3698
|
+
return db.all(`SELECT MIN(pr.value) AS value, COUNT(DISTINCT pr.page_id) AS documents
|
|
3699
|
+
FROM properties pr JOIN pages p ON p.id = pr.page_id
|
|
3700
|
+
WHERE p.generation = ? AND pr.key_lc = ? AND pr.value IS NOT NULL
|
|
3701
|
+
GROUP BY pr.value_lc
|
|
3702
|
+
ORDER BY documents DESC, pr.value_lc
|
|
3703
|
+
LIMIT ?`, [
|
|
3704
|
+
activeIndexGeneration(db),
|
|
3705
|
+
key.toLowerCase(),
|
|
3706
|
+
limit
|
|
3707
|
+
]);
|
|
3708
|
+
}
|
|
3709
|
+
/**
|
|
3710
|
+
* Every document that passes all `filters`, in name order, with the values of the keys the
|
|
3711
|
+
* positive filters name (not `title`, which the row already shows). Empty without a filter:
|
|
3712
|
+
* listing the whole graph is [[All Documents]]' job.
|
|
3713
|
+
*/
|
|
3714
|
+
function documentsMatchingProperties(db, filters, limit = PROPERTY_MATCH_CAP) {
|
|
3715
|
+
if (filters.length === 0) return [];
|
|
3716
|
+
const narrow = propertyFilterSql(filters);
|
|
3717
|
+
const pages = db.all(`SELECT p.id AS id, p.concept AS concept, p.kind AS kind FROM pages p
|
|
3718
|
+
WHERE p.generation = ?${narrow.sql}
|
|
3719
|
+
ORDER BY p.concept_key
|
|
3720
|
+
LIMIT ?`, [
|
|
3721
|
+
activeIndexGeneration(db),
|
|
3722
|
+
...narrow.params,
|
|
3723
|
+
limit
|
|
3724
|
+
]);
|
|
3725
|
+
if (pages.length === 0) return [];
|
|
3726
|
+
const shown = [...new Set(filters.filter((f) => !f.negated).map((f) => f.key.toLowerCase()))].filter((key) => key !== "title");
|
|
3727
|
+
const values = /* @__PURE__ */ new Map();
|
|
3728
|
+
if (shown.length > 0) {
|
|
3729
|
+
const rows = db.all(`SELECT page_id, key, value FROM properties
|
|
3730
|
+
WHERE page_id IN (SELECT value FROM json_each(?))
|
|
3731
|
+
AND key_lc IN (SELECT value FROM json_each(?))
|
|
3732
|
+
AND value IS NOT NULL
|
|
3733
|
+
ORDER BY rowid`, [JSON.stringify(pages.map((page) => page.id)), JSON.stringify(shown)]);
|
|
3734
|
+
for (const row of rows) {
|
|
3735
|
+
const list = values.get(row.page_id);
|
|
3736
|
+
const entry = {
|
|
3737
|
+
key: row.key,
|
|
3738
|
+
value: row.value
|
|
3739
|
+
};
|
|
3740
|
+
if (list) list.push(entry);
|
|
3741
|
+
else values.set(row.page_id, [entry]);
|
|
3742
|
+
}
|
|
3743
|
+
}
|
|
3744
|
+
return pages.map((page) => ({
|
|
3745
|
+
concept: page.concept,
|
|
3746
|
+
kind: page.kind,
|
|
3747
|
+
properties: values.get(page.id) ?? []
|
|
3748
|
+
}));
|
|
3749
|
+
}
|
|
3398
3750
|
/** Escape a literal for a LIKE pattern, so an asset name can never behave as a wildcard. */
|
|
3399
3751
|
function likeLiteral(needle) {
|
|
3400
3752
|
return needle.replace(/[\\%_]/g, (c) => `\\${c}`);
|
|
@@ -3918,7 +4270,7 @@ function cacheRoot(env) {
|
|
|
3918
4270
|
* recovered from a joined path (see there).
|
|
3919
4271
|
*/
|
|
3920
4272
|
var CACHE_FILE_NAME = `local-cache.v${CACHE_DB_VERSION}.bin`;
|
|
3921
|
-
var INDEX_FILE_NAME = `index.
|
|
4273
|
+
var INDEX_FILE_NAME = `index.v14.sqlite`;
|
|
3922
4274
|
var VECTORS_FILE_NAME = `vectors.v1.sqlite`;
|
|
3923
4275
|
function cacheFile(dir) {
|
|
3924
4276
|
return join(dir, CACHE_FILE_NAME);
|
|
@@ -4244,6 +4596,65 @@ async function removeCacheRoot(env) {
|
|
|
4244
4596
|
force: true
|
|
4245
4597
|
});
|
|
4246
4598
|
}
|
|
4599
|
+
var GRAPH_ID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
4600
|
+
/**
|
|
4601
|
+
* The account a synced graph's cache was written for. Cache directories are keyed by server host
|
|
4602
|
+
* and graph id, not by account, and two agents with their own `ETHERPK_MCP_CONFIG` can sign in as
|
|
4603
|
+
* different accounts on one server and share the cache root. The stamp is what lets one of them
|
|
4604
|
+
* tidy only its own caches.
|
|
4605
|
+
*/
|
|
4606
|
+
var OWNER_FILE = "owner.json";
|
|
4607
|
+
/** Record which account's graph this cache holds. Written when a synced graph is opened. */
|
|
4608
|
+
async function stampGraphCacheOwner(dir, principalId) {
|
|
4609
|
+
await mkdir(dir, {
|
|
4610
|
+
recursive: true,
|
|
4611
|
+
mode: 448
|
|
4612
|
+
});
|
|
4613
|
+
await writeFile(join(dir, OWNER_FILE), `${JSON.stringify({ principalId })}\n`, { mode: 384 });
|
|
4614
|
+
}
|
|
4615
|
+
/** The account a cache directory was stamped for, or null (none, or unreadable). */
|
|
4616
|
+
async function graphCacheOwner(dir) {
|
|
4617
|
+
try {
|
|
4618
|
+
const { principalId } = JSON.parse(await readFile(join(dir, OWNER_FILE), "utf8"));
|
|
4619
|
+
return typeof principalId === "string" ? principalId : null;
|
|
4620
|
+
} catch {
|
|
4621
|
+
return null;
|
|
4622
|
+
}
|
|
4623
|
+
}
|
|
4624
|
+
/**
|
|
4625
|
+
* Remove this account's cached graphs of one server that the server no longer lists for it: a
|
|
4626
|
+
* graph deleted, left or taken away since this machine last served it. Its cache is the graph's
|
|
4627
|
+
* plaintext, and without this it would stay on disk until `logout`. `listed` must be the
|
|
4628
|
+
* server's own answer: a failed listing must not reach here, or every cached graph would go.
|
|
4629
|
+
*
|
|
4630
|
+
* Only directories named as a graph id and stamped for `principalId` are touched. Another
|
|
4631
|
+
* account's cache under the same root (its unacknowledged edits included) is not this account's
|
|
4632
|
+
* to drop, and a cache written before the stamp existed is left for `logout`. Returns the ids
|
|
4633
|
+
* removed.
|
|
4634
|
+
*/
|
|
4635
|
+
async function removeUnlistedGraphCaches(env, serverBaseUrl, principalId, listed) {
|
|
4636
|
+
const keep = new Set([...listed].map((id) => id.toLowerCase()));
|
|
4637
|
+
const hostDir = dirname(graphCacheDir(env, serverBaseUrl, "x"));
|
|
4638
|
+
let entries;
|
|
4639
|
+
try {
|
|
4640
|
+
entries = await readdir(hostDir, { withFileTypes: true });
|
|
4641
|
+
} catch (error) {
|
|
4642
|
+
if (error.code === "ENOENT") return [];
|
|
4643
|
+
throw error;
|
|
4644
|
+
}
|
|
4645
|
+
const removed = [];
|
|
4646
|
+
for (const entry of entries) {
|
|
4647
|
+
if (!entry.isDirectory() || !GRAPH_ID.test(entry.name) || keep.has(entry.name.toLowerCase())) continue;
|
|
4648
|
+
const dir = join(hostDir, entry.name);
|
|
4649
|
+
if (await graphCacheOwner(dir) !== principalId) continue;
|
|
4650
|
+
await rm(dir, {
|
|
4651
|
+
recursive: true,
|
|
4652
|
+
force: true
|
|
4653
|
+
});
|
|
4654
|
+
removed.push(entry.name);
|
|
4655
|
+
}
|
|
4656
|
+
return removed;
|
|
4657
|
+
}
|
|
4247
4658
|
/** Remove one server's persisted graphs (logout of that server while others stay). */
|
|
4248
4659
|
async function removeServerCache(env, serverBaseUrl) {
|
|
4249
4660
|
await rm(dirname(graphCacheDir(env, serverBaseUrl, "x")), {
|
|
@@ -4733,6 +5144,20 @@ var entitlementLimitsSchema = z.strictObject({
|
|
|
4733
5144
|
assetBytes: z.int().nonnegative(),
|
|
4734
5145
|
assetChunks: z.int().nonnegative()
|
|
4735
5146
|
});
|
|
5147
|
+
/**
|
|
5148
|
+
* Set, and only ever `true`, while the Billing Account's subscription has a payment outstanding
|
|
5149
|
+
* (Stripe `past_due` or `unpaid`). It explains a `grace` or `read_only` status as a failed
|
|
5150
|
+
* payment rather than an ended plan, so the Client can say "fix your payment" instead of "your
|
|
5151
|
+
* subscription has ended". Issuers omit it otherwise, which keeps every other statement readable
|
|
5152
|
+
* by a Sync Server whose strict schema predates the field.
|
|
5153
|
+
*/
|
|
5154
|
+
var paymentOverdueSchema = z.literal(true).optional();
|
|
5155
|
+
/**
|
|
5156
|
+
* When the Billing Account's trial ends, set only while the trial runs, so the Client and the Sync
|
|
5157
|
+
* portal can say "Trial, ends 7 Oct" as the Billing page does. Omitted otherwise, as
|
|
5158
|
+
* `paymentOverdue` is.
|
|
5159
|
+
*/
|
|
5160
|
+
var trialEndsAtSchema = z.iso.datetime({ offset: true }).optional();
|
|
4736
5161
|
var serviceEntitlementSchema = z.strictObject({
|
|
4737
5162
|
eventId: z.uuid(),
|
|
4738
5163
|
issuer: httpsUrl,
|
|
@@ -4749,6 +5174,8 @@ var serviceEntitlementSchema = z.strictObject({
|
|
|
4749
5174
|
]),
|
|
4750
5175
|
plan: z.string().min(1).max(64),
|
|
4751
5176
|
limits: entitlementLimitsSchema,
|
|
5177
|
+
paymentOverdue: paymentOverdueSchema,
|
|
5178
|
+
trialEndsAt: trialEndsAtSchema,
|
|
4752
5179
|
effectiveAt: z.iso.datetime({ offset: true }),
|
|
4753
5180
|
expiresAt: z.iso.datetime({ offset: true })
|
|
4754
5181
|
}).refine(({ effectiveAt, expiresAt }) => Date.parse(expiresAt) > Date.parse(effectiveAt), {
|
|
@@ -4774,14 +5201,18 @@ z.strictObject({
|
|
|
4774
5201
|
id: z.uuid(),
|
|
4775
5202
|
email: z.email().nullable(),
|
|
4776
5203
|
name: z.string().min(1).nullable(),
|
|
4777
|
-
image: httpsUrl.nullable()
|
|
5204
|
+
image: httpsUrl.nullable(),
|
|
5205
|
+
emailVerified: z.boolean().optional()
|
|
4778
5206
|
}),
|
|
4779
5207
|
authentication: syncAuthenticationSchema,
|
|
5208
|
+
invitesNeedVerifiedEmail: z.boolean().optional(),
|
|
4780
5209
|
clientUrl: httpsUrl.optional(),
|
|
4781
5210
|
entitlement: z.strictObject({
|
|
4782
5211
|
plan: z.string().min(1).max(64),
|
|
4783
5212
|
status: serviceEntitlementSchema.shape.status,
|
|
4784
5213
|
limits: entitlementLimitsSchema,
|
|
5214
|
+
paymentOverdue: paymentOverdueSchema,
|
|
5215
|
+
trialEndsAt: trialEndsAtSchema,
|
|
4785
5216
|
usage: z.strictObject({
|
|
4786
5217
|
ownedGraphs: z.int().nonnegative(),
|
|
4787
5218
|
ownedStorageBytes: z.int().nonnegative()
|
|
@@ -4803,6 +5234,19 @@ var quotaErrorResponseSchema = z.strictObject({
|
|
|
4803
5234
|
ownerPrincipalId: z.uuid().optional(),
|
|
4804
5235
|
retryable: z.literal(true)
|
|
4805
5236
|
});
|
|
5237
|
+
z.strictObject({
|
|
5238
|
+
eventId: z.uuid(),
|
|
5239
|
+
issuer: httpsUrl,
|
|
5240
|
+
audience: z.literal("urn:etherpk:sync-identity"),
|
|
5241
|
+
subject: z.string().min(1).max(256),
|
|
5242
|
+
revision: z.int().positive(),
|
|
5243
|
+
disabled: z.boolean(),
|
|
5244
|
+
deleted: z.boolean(),
|
|
5245
|
+
credentialsRevokedAt: z.iso.datetime({ offset: true }).nullable(),
|
|
5246
|
+
email: z.email().optional(),
|
|
5247
|
+
emailVerified: z.boolean().optional(),
|
|
5248
|
+
issuedAt: z.iso.datetime({ offset: true })
|
|
5249
|
+
});
|
|
4806
5250
|
var SYNC_PROTOCOL_LIMITS = Object.freeze({
|
|
4807
5251
|
maxMessageBytes: 12 * 1024 * 1024,
|
|
4808
5252
|
maxEnvelopeBytes: 8 * 1024 * 1024,
|
|
@@ -5015,6 +5459,7 @@ function serverMessageSchema(limits) {
|
|
|
5015
5459
|
code: z.enum(SYNC_ERROR_CODES),
|
|
5016
5460
|
message: z.string().min(1).max(256),
|
|
5017
5461
|
docId: uuid.optional(),
|
|
5462
|
+
outboxId: uuid.optional(),
|
|
5018
5463
|
currentGeneration: generation.optional(),
|
|
5019
5464
|
quotaCode: quotaErrorCodeSchema.optional(),
|
|
5020
5465
|
retryable: z.boolean().optional()
|
|
@@ -5169,6 +5614,9 @@ var ASSET_CHUNK_PLAINTEXT_BYTES = 4 * 1024 * 1024;
|
|
|
5169
5614
|
function assetChunkCount(size) {
|
|
5170
5615
|
return Math.max(1, Math.ceil(size / ASSET_CHUNK_PLAINTEXT_BYTES));
|
|
5171
5616
|
}
|
|
5617
|
+
var DAY = 24 * (60 * 6e4);
|
|
5618
|
+
365 * DAY, 30 * DAY, 7 * DAY;
|
|
5619
|
+
Number.MAX_SAFE_INTEGER;
|
|
5172
5620
|
//#endregion
|
|
5173
5621
|
//#region ../client/src/lib/sync/state-vector.ts
|
|
5174
5622
|
/**
|
|
@@ -5210,6 +5658,10 @@ var REMOTE = Symbol("etherpk-remote");
|
|
|
5210
5658
|
var CACHE_SEED = Symbol("etherpk-cache-seed");
|
|
5211
5659
|
/** Structural lifecycle changes which must never be mistaken for a user's edit. */
|
|
5212
5660
|
var SUPPRESSED = Symbol("etherpk-suppressed");
|
|
5661
|
+
/** Health that says the bytes are not this document's, so neither text nor emptiness is real. */
|
|
5662
|
+
function contentBlocked(health) {
|
|
5663
|
+
return health === "key-unavailable" || health === "ciphertext-corrupt";
|
|
5664
|
+
}
|
|
5213
5665
|
function newSyncCompletion() {
|
|
5214
5666
|
let resolvePromise;
|
|
5215
5667
|
let rejectPromise;
|
|
@@ -5296,6 +5748,10 @@ function createDocSync(deps) {
|
|
|
5296
5748
|
let pendingSnapshot;
|
|
5297
5749
|
/** A read-back failed: the next idle uploads a fresh snapshot whatever the tail size. */
|
|
5298
5750
|
let snapshotRetryWanted = false;
|
|
5751
|
+
/** Tell the graph the queue changed; see `DocSyncDeps.onQueueChange`. */
|
|
5752
|
+
function queueChanged() {
|
|
5753
|
+
if (!destroyed) deps.onQueueChange?.();
|
|
5754
|
+
}
|
|
5299
5755
|
function requestCatchup(afterSeq) {
|
|
5300
5756
|
if (!catchupActive) {
|
|
5301
5757
|
if (catchupCompletion.settled) catchupCompletion = newSyncCompletion();
|
|
@@ -5382,6 +5838,7 @@ function createDocSync(deps) {
|
|
|
5382
5838
|
}
|
|
5383
5839
|
async function transmit(operation) {
|
|
5384
5840
|
await persistTask(() => persist.markAttempt(operation.outboxId, now()));
|
|
5841
|
+
if (durableQueue[0]?.outboxId !== operation.outboxId) return;
|
|
5385
5842
|
if (operation.kind === "delete") {
|
|
5386
5843
|
send({
|
|
5387
5844
|
type: "delete",
|
|
@@ -5442,6 +5899,7 @@ function createDocSync(deps) {
|
|
|
5442
5899
|
lastAttemptAt: null
|
|
5443
5900
|
};
|
|
5444
5901
|
durableQueue.push(durable);
|
|
5902
|
+
queueChanged();
|
|
5445
5903
|
if (operation.kind === "resurrect") lifecycleState = "resurrecting";
|
|
5446
5904
|
await transmit(durable);
|
|
5447
5905
|
}
|
|
@@ -5695,6 +6153,7 @@ function createDocSync(deps) {
|
|
|
5695
6153
|
baseline.destroy();
|
|
5696
6154
|
} else lastSyncedSnapshot = Y.snapshot(doc);
|
|
5697
6155
|
if (lifecycleState === "seeding") lifecycleState = "active";
|
|
6156
|
+
queueChanged();
|
|
5698
6157
|
},
|
|
5699
6158
|
async receive(message) {
|
|
5700
6159
|
switch (message.type) {
|
|
@@ -5730,6 +6189,7 @@ function createDocSync(deps) {
|
|
|
5730
6189
|
scheduleAutomaticCompaction();
|
|
5731
6190
|
}
|
|
5732
6191
|
durableQueue.shift();
|
|
6192
|
+
queueChanged();
|
|
5733
6193
|
send({
|
|
5734
6194
|
type: "ack_confirm",
|
|
5735
6195
|
outboxId: message.outboxId
|
|
@@ -5760,6 +6220,7 @@ function createDocSync(deps) {
|
|
|
5760
6220
|
}
|
|
5761
6221
|
await persist.purge();
|
|
5762
6222
|
durableQueue = [];
|
|
6223
|
+
queueChanged();
|
|
5763
6224
|
boundarySnapshots.clear();
|
|
5764
6225
|
recoveredDirtyTokens.clear();
|
|
5765
6226
|
activeDirtyToken = void 0;
|
|
@@ -5825,6 +6286,7 @@ function createDocSync(deps) {
|
|
|
5825
6286
|
seeding = false;
|
|
5826
6287
|
}
|
|
5827
6288
|
for (const operation of durableQueue.splice(0)) await persist.discard(operation.outboxId);
|
|
6289
|
+
queueChanged();
|
|
5828
6290
|
boundarySnapshots.clear();
|
|
5829
6291
|
lastSyncedSnapshot = Y.snapshot(doc);
|
|
5830
6292
|
const operation = {
|
|
@@ -5850,6 +6312,7 @@ function createDocSync(deps) {
|
|
|
5850
6312
|
attemptCount: 0,
|
|
5851
6313
|
lastAttemptAt: null
|
|
5852
6314
|
});
|
|
6315
|
+
queueChanged();
|
|
5853
6316
|
await transmit(durableQueue[0]);
|
|
5854
6317
|
},
|
|
5855
6318
|
lifecycle: () => lifecycleState,
|
|
@@ -5857,6 +6320,7 @@ function createDocSync(deps) {
|
|
|
5857
6320
|
async staleGeneration(currentGeneration) {
|
|
5858
6321
|
syncHealth = "generation-stale";
|
|
5859
6322
|
for (const operation of durableQueue.splice(0)) await persist.discard(operation.outboxId);
|
|
6323
|
+
queueChanged();
|
|
5860
6324
|
boundarySnapshots.clear();
|
|
5861
6325
|
documentGeneration = currentGeneration;
|
|
5862
6326
|
lastSeq = 0;
|
|
@@ -5877,6 +6341,12 @@ function createDocSync(deps) {
|
|
|
5877
6341
|
await sendAwareness([doc.clientID]);
|
|
5878
6342
|
},
|
|
5879
6343
|
isIdle: () => durableQueue.length === 0 && !debounceTimer && !drainPromise,
|
|
6344
|
+
unsentOperations: () => durableQueue.length,
|
|
6345
|
+
retryUnsent() {
|
|
6346
|
+
if (destroyed) return;
|
|
6347
|
+
if (durableQueue[0]) detached(transmit(durableQueue[0]));
|
|
6348
|
+
else detached(requestDrain());
|
|
6349
|
+
},
|
|
5880
6350
|
destroy() {
|
|
5881
6351
|
destroyed = true;
|
|
5882
6352
|
boundarySnapshots.clear();
|
|
@@ -6066,6 +6536,29 @@ function createPresenceSession(identity, options = {}) {
|
|
|
6066
6536
|
}
|
|
6067
6537
|
};
|
|
6068
6538
|
}
|
|
6539
|
+
var RECONNECT_CAP_MS = 3e4;
|
|
6540
|
+
/** The wait before reconnect attempt `attempt` (1 for the first retry). */
|
|
6541
|
+
function reconnectDelayMs(attempt, random = Math.random) {
|
|
6542
|
+
const exponent = Math.min(Math.max(0, attempt - 1), 16);
|
|
6543
|
+
const ceiling = Math.min(RECONNECT_CAP_MS, 500 * 2 ** exponent);
|
|
6544
|
+
return Math.floor(random() * ceiling);
|
|
6545
|
+
}
|
|
6546
|
+
/**
|
|
6547
|
+
* How long a graph waits before sending a write the Sync Server refused on a quota again: a
|
|
6548
|
+
* lapsed plan, a storage allowance, or a plan the server could not confirm. The refusal is an
|
|
6549
|
+
* answer, not an outage, so the schedule is slower than a reconnect's: the ceiling doubles from
|
|
6550
|
+
* ten seconds to two minutes. Half of it is fixed and half drawn below it, so a refused client
|
|
6551
|
+
* is never back within milliseconds and a crowd refused together spreads out. `round` counts
|
|
6552
|
+
* the retries already made for this refusal, from 1. The workspace also retries at once when the
|
|
6553
|
+
* person comes back to the tab, which is when a restarted plan usually shows.
|
|
6554
|
+
*/
|
|
6555
|
+
var REFUSAL_RETRY_BASE_MS = 1e4;
|
|
6556
|
+
var REFUSAL_RETRY_CAP_MS = 12e4;
|
|
6557
|
+
function refusalRetryDelayMs(round, random = Math.random) {
|
|
6558
|
+
const exponent = Math.min(Math.max(0, round - 1), 16);
|
|
6559
|
+
const ceiling = Math.min(REFUSAL_RETRY_CAP_MS, REFUSAL_RETRY_BASE_MS * 2 ** exponent);
|
|
6560
|
+
return Math.floor(ceiling / 2 + random() * (ceiling / 2));
|
|
6561
|
+
}
|
|
6069
6562
|
//#endregion
|
|
6070
6563
|
//#region ../client/src/lib/document/quick-notes.ts
|
|
6071
6564
|
/** A hard ceiling, so a corrupt or hostile list cannot swamp the View or the root doc. */
|
|
@@ -6163,6 +6656,172 @@ function parseFrontmatter(text) {
|
|
|
6163
6656
|
}
|
|
6164
6657
|
}
|
|
6165
6658
|
//#endregion
|
|
6659
|
+
//#region ../client/src/lib/document/frontmatter/frontmatter-yaml.ts
|
|
6660
|
+
/**
|
|
6661
|
+
* Reading, editing and writing a [[Frontmatter]] block's YAML in the one style EtherPK writes
|
|
6662
|
+
* (ADR 0108).
|
|
6663
|
+
*
|
|
6664
|
+
* Every writer goes through here: the identity write-back, publishing, the arbitrary-key patch,
|
|
6665
|
+
* the importers, Add frontmatter and the editor's tidy when an editing episode ends. Edits go
|
|
6666
|
+
* through the `yaml` library's document model rather than a plain object, so a comment, and any
|
|
6667
|
+
* key the edit does not touch, survives. The block is then written in the EtherPK style:
|
|
6668
|
+
*
|
|
6669
|
+
* - two-space indentation, a list under its key as ` - item`, one item per line;
|
|
6670
|
+
* - an inline list (`[a, b]`) written as a block list; an empty `[]` written bare on a key EtherPK
|
|
6671
|
+
* reads (`publications:`), where empty means not set, and kept on any other key;
|
|
6672
|
+
* - quotes only where a value needs them (`"true"` stays quoted, `"hello"` does not);
|
|
6673
|
+
* - no line wrapping, no blank lines, key order as found, comments kept;
|
|
6674
|
+
* - an empty value written bare (`slug:`), which reads as "fill this in".
|
|
6675
|
+
*
|
|
6676
|
+
* Restyling never changes what the block says: the value is compared before and after, and a
|
|
6677
|
+
* rewrite that would change it is dropped. A block that does not parse is never rewritten.
|
|
6678
|
+
*
|
|
6679
|
+
* Pure: no store, no editor. What counts as a block comes from `frontmatter-span.ts`.
|
|
6680
|
+
*/
|
|
6681
|
+
/** The `yaml` serialiser options that make up the EtherPK style. */
|
|
6682
|
+
var STYLE = {
|
|
6683
|
+
indent: 2,
|
|
6684
|
+
indentSeq: true,
|
|
6685
|
+
lineWidth: 0,
|
|
6686
|
+
minContentWidth: 0,
|
|
6687
|
+
nullStr: ""
|
|
6688
|
+
};
|
|
6689
|
+
/**
|
|
6690
|
+
* The block's YAML parsed as a document, or null when it does not parse or is not a mapping.
|
|
6691
|
+
* An empty block (or one holding only comments) is an empty mapping.
|
|
6692
|
+
*/
|
|
6693
|
+
function parseBlock(body) {
|
|
6694
|
+
const doc = parseDocument(body);
|
|
6695
|
+
if (doc.errors.length > 0) return null;
|
|
6696
|
+
if (doc.contents === null) {
|
|
6697
|
+
doc.contents = new YAMLMap();
|
|
6698
|
+
return doc;
|
|
6699
|
+
}
|
|
6700
|
+
return isMap(doc.contents) ? doc : null;
|
|
6701
|
+
}
|
|
6702
|
+
/** The block's YAML as a plain object, or null when it does not parse to a mapping. */
|
|
6703
|
+
function frontmatterData(body) {
|
|
6704
|
+
const doc = parseBlock(body);
|
|
6705
|
+
return doc === null ? null : doc.toJS();
|
|
6706
|
+
}
|
|
6707
|
+
/**
|
|
6708
|
+
* The keys EtherPK reads, on which an empty value means the key is not set (ADR 0108, point 5).
|
|
6709
|
+
* On these an empty `[]` and a bare key say the same thing, so the style writes it bare. Another
|
|
6710
|
+
* key's `[]` is kept: to a tool that reads it, an empty list and no value may differ.
|
|
6711
|
+
*/
|
|
6712
|
+
var NOT_SET_WHEN_EMPTY = new Set([
|
|
6713
|
+
"title",
|
|
6714
|
+
"aliases",
|
|
6715
|
+
"public",
|
|
6716
|
+
"publications",
|
|
6717
|
+
"slug",
|
|
6718
|
+
"date",
|
|
6719
|
+
"publication"
|
|
6720
|
+
]);
|
|
6721
|
+
/** Apply the EtherPK style to a document in place. */
|
|
6722
|
+
function applyStyle(doc) {
|
|
6723
|
+
visit(doc, {
|
|
6724
|
+
Pair(_, pair, path) {
|
|
6725
|
+
if (path.length !== 2) return;
|
|
6726
|
+
const key = isScalar(pair.key) ? pair.key.value : pair.key;
|
|
6727
|
+
if (typeof key === "string" && NOT_SET_WHEN_EMPTY.has(key) && isSeq(pair.value) && pair.value.items.length === 0) pair.value = new Scalar(null);
|
|
6728
|
+
},
|
|
6729
|
+
Scalar(_, node) {
|
|
6730
|
+
node.spaceBefore = false;
|
|
6731
|
+
if (typeof node.value === "string" && !node.value.includes("\n") && (node.type === "QUOTE_DOUBLE" || node.type === "QUOTE_SINGLE")) node.type = "PLAIN";
|
|
6732
|
+
},
|
|
6733
|
+
Map(_, node) {
|
|
6734
|
+
node.spaceBefore = false;
|
|
6735
|
+
if (node.items.length > 0) node.flow = false;
|
|
6736
|
+
},
|
|
6737
|
+
Seq(_, node) {
|
|
6738
|
+
node.spaceBefore = false;
|
|
6739
|
+
if (node.items.length > 0) node.flow = false;
|
|
6740
|
+
}
|
|
6741
|
+
});
|
|
6742
|
+
}
|
|
6743
|
+
/** The styled YAML of a document, ending in a newline, or '' when it has no keys and no comment. */
|
|
6744
|
+
function styledYaml(doc) {
|
|
6745
|
+
applyStyle(doc);
|
|
6746
|
+
const map = doc.contents;
|
|
6747
|
+
if (isMap(map) && map.items.length === 0) return doc.commentBefore ? `${doc.commentBefore.replace(/^/gm, "#")}\n` : "";
|
|
6748
|
+
return doc.toString(STYLE);
|
|
6749
|
+
}
|
|
6750
|
+
/** Deep equality over the plain values a block parses to. */
|
|
6751
|
+
function sameValue(a, b) {
|
|
6752
|
+
return JSON.stringify(a) === JSON.stringify(b);
|
|
6753
|
+
}
|
|
6754
|
+
/**
|
|
6755
|
+
* A new block holding `data`, in the EtherPK style: `---\n…---\n`, or '' when `data` has no keys.
|
|
6756
|
+
* For a document that has no block yet; to change a block that exists use {@link editFrontmatter},
|
|
6757
|
+
* which keeps what the block already says.
|
|
6758
|
+
*/
|
|
6759
|
+
function renderFrontmatter(data) {
|
|
6760
|
+
if (Object.keys(data).length === 0) return "";
|
|
6761
|
+
return `---\n${styledYaml(new Document(data))}---\n`;
|
|
6762
|
+
}
|
|
6763
|
+
/** The line terminator a block uses, so a rewrite does not mix endings in a CRLF file. */
|
|
6764
|
+
function eolOf(text, end) {
|
|
6765
|
+
return text.slice(0, end).includes("\r\n") ? "\r\n" : "\n";
|
|
6766
|
+
}
|
|
6767
|
+
function withEol(yaml, eol) {
|
|
6768
|
+
return eol === "\n" ? yaml : yaml.replace(/\n/g, eol);
|
|
6769
|
+
}
|
|
6770
|
+
function editorFor(doc) {
|
|
6771
|
+
const map = doc.contents;
|
|
6772
|
+
const keyOf = (item) => String(isScalar(item.key) ? item.key.value : item.key);
|
|
6773
|
+
return {
|
|
6774
|
+
has: (key) => map.has(key),
|
|
6775
|
+
get: (key) => map.has(key) ? doc.toJS()[key] : void 0,
|
|
6776
|
+
set(key, value, where = "last") {
|
|
6777
|
+
if (map.has(key)) {
|
|
6778
|
+
const current = doc.toJS()[key];
|
|
6779
|
+
if (sameValue(current, value)) return;
|
|
6780
|
+
map.set(key, doc.createNode(value));
|
|
6781
|
+
return;
|
|
6782
|
+
}
|
|
6783
|
+
const pair = doc.createPair(key, value);
|
|
6784
|
+
if (where === "first") map.items.unshift(pair);
|
|
6785
|
+
else map.items.push(pair);
|
|
6786
|
+
},
|
|
6787
|
+
delete: (key) => {
|
|
6788
|
+
map.delete(key);
|
|
6789
|
+
},
|
|
6790
|
+
keys: () => map.items.map(keyOf)
|
|
6791
|
+
};
|
|
6792
|
+
}
|
|
6793
|
+
/**
|
|
6794
|
+
* The text with its block changed by `edit`, then written in the EtherPK style. The same string
|
|
6795
|
+
* comes back when the edit changes nothing the block says, so a writer that runs on every save
|
|
6796
|
+
* never restyles a block it agrees with. A block whose YAML does not parse is left alone: rewriting
|
|
6797
|
+
* it would destroy whatever the person was typing. A block the edit empties of every key is
|
|
6798
|
+
* removed.
|
|
6799
|
+
*/
|
|
6800
|
+
function editFrontmatter(text, edit, options = {}) {
|
|
6801
|
+
const span = frontmatterSpan(text);
|
|
6802
|
+
if (!span && !options.addBlock) return text;
|
|
6803
|
+
const doc = span ? parseBlock(span.body) : parseBlock("");
|
|
6804
|
+
if (doc === null) return text;
|
|
6805
|
+
const before = doc.toJS();
|
|
6806
|
+
edit(editorFor(doc));
|
|
6807
|
+
if (sameValue(doc.toJS(), before)) return text;
|
|
6808
|
+
const body = span ? text.slice(span.end) : text;
|
|
6809
|
+
if (doc.contents.items.length === 0) return body;
|
|
6810
|
+
const eol = span ? eolOf(text, span.end) : "\n";
|
|
6811
|
+
return `---${eol}${withEol(styledYaml(doc), eol)}---${eol}${body}`;
|
|
6812
|
+
}
|
|
6813
|
+
/**
|
|
6814
|
+
* Whether a value counts as not set (ADR 0108): nothing, a blank string, an empty list, or a
|
|
6815
|
+
* mapping whose every value is itself empty. `false` and `0` are values.
|
|
6816
|
+
*/
|
|
6817
|
+
function isEmptyValue(value) {
|
|
6818
|
+
if (value === null || value === void 0) return true;
|
|
6819
|
+
if (typeof value === "string") return value.trim() === "";
|
|
6820
|
+
if (Array.isArray(value)) return value.length === 0;
|
|
6821
|
+
if (typeof value === "object" && !(value instanceof Date)) return Object.values(value).every(isEmptyValue);
|
|
6822
|
+
return false;
|
|
6823
|
+
}
|
|
6824
|
+
//#endregion
|
|
6166
6825
|
//#region ../client/src/lib/document/publish/publication.ts
|
|
6167
6826
|
/** A publication id: kebab-case, what `publications:` entries and `theme:` graph ids look like. */
|
|
6168
6827
|
var PUBLICATION_ID = /^[a-z0-9]+(?:-[a-z0-9]+)*$/;
|
|
@@ -6178,13 +6837,19 @@ function readMembership(text, concept) {
|
|
|
6178
6837
|
const { data } = parseFrontmatter(text);
|
|
6179
6838
|
const issues = [];
|
|
6180
6839
|
const isPublic = data.public === true;
|
|
6840
|
+
if (!isEmptyValue(data.public) && typeof data.public !== "boolean") issues.push({
|
|
6841
|
+
level: "warning",
|
|
6842
|
+
code: "public-not-a-boolean",
|
|
6843
|
+
message: `\`public\` is the text "${String(data.public)}", not \`true\` or \`false\`, so the document is not public. Write \`public: true\` without quotes to publish it.`,
|
|
6844
|
+
concept
|
|
6845
|
+
});
|
|
6181
6846
|
const publications = [];
|
|
6182
|
-
const raw = data.publications;
|
|
6183
|
-
const entries = raw === void 0 ? [] : Array.isArray(raw) ? raw :
|
|
6847
|
+
const raw = isEmptyValue(data.publications) ? void 0 : data.publications;
|
|
6848
|
+
const entries = raw === void 0 ? [] : Array.isArray(raw) ? raw : null;
|
|
6184
6849
|
if (entries === null) issues.push({
|
|
6185
6850
|
level: "warning",
|
|
6186
6851
|
code: "publications-not-a-list",
|
|
6187
|
-
message: "`publications` must be a list of publication ids,
|
|
6852
|
+
message: "`publications` must be a list of publication ids, each on its own line under the key as `- docs`, even when there is one.",
|
|
6188
6853
|
concept
|
|
6189
6854
|
});
|
|
6190
6855
|
else for (const entry of entries) if (isPublicationId(entry)) {
|
|
@@ -6201,13 +6866,33 @@ function readMembership(text, concept) {
|
|
|
6201
6866
|
issues
|
|
6202
6867
|
};
|
|
6203
6868
|
}
|
|
6869
|
+
var DATE = /^\d{4}-\d{2}-\d{2}$/;
|
|
6870
|
+
/**
|
|
6871
|
+
* The day a page is dated on a blog, from its `date:` key, and a warning when the key holds
|
|
6872
|
+
* something that is not a calendar day. A journal entry is dated by its name, never by this key.
|
|
6873
|
+
*/
|
|
6874
|
+
function readPageDate(doc) {
|
|
6875
|
+
const date = parseFrontmatter(doc.text).data.date;
|
|
6876
|
+
if (isEmptyValue(date)) return { issues: [] };
|
|
6877
|
+
const text = date instanceof Date ? date.toISOString().slice(0, 10) : String(date);
|
|
6878
|
+
if (DATE.test(text)) return {
|
|
6879
|
+
date: text,
|
|
6880
|
+
issues: []
|
|
6881
|
+
};
|
|
6882
|
+
return { issues: [{
|
|
6883
|
+
level: "warning",
|
|
6884
|
+
code: "invalid-date",
|
|
6885
|
+
message: `"${doc.concept}" has \`date: ${String(date)}\`, which is not a calendar day (YYYY-MM-DD); the document is undated.`,
|
|
6886
|
+
concept: doc.concept
|
|
6887
|
+
}] };
|
|
6888
|
+
}
|
|
6204
6889
|
var KINDS$1 = ["docs", "blog"];
|
|
6205
6890
|
var SELECTIONS = ["named", "all-public"];
|
|
6206
6891
|
/** The publication a page defines through its `publication:` mapping, validated. */
|
|
6207
6892
|
function readPublicationDefinition(doc) {
|
|
6208
6893
|
const { data, body } = parseFrontmatter(doc.text);
|
|
6209
6894
|
const raw = data.publication;
|
|
6210
|
-
if (raw
|
|
6895
|
+
if (isEmptyValue(raw)) return {
|
|
6211
6896
|
publication: null,
|
|
6212
6897
|
issues: []
|
|
6213
6898
|
};
|
|
@@ -6232,7 +6917,7 @@ function readPublicationDefinition(doc) {
|
|
|
6232
6917
|
issues
|
|
6233
6918
|
};
|
|
6234
6919
|
}
|
|
6235
|
-
const def = raw;
|
|
6920
|
+
const def = Object.fromEntries(Object.entries(raw).filter(([, value]) => !isEmptyValue(value)));
|
|
6236
6921
|
const id = def.id;
|
|
6237
6922
|
if (id === void 0 || id === null || id === "") issue("publication-missing-id", "`publication` needs an `id`: lower-case letters, digits and hyphens, like `docs`.");
|
|
6238
6923
|
else if (!isPublicationId(id)) issue("publication-invalid-id", `"${String(id)}" is not a publication id: use lower-case letters, digits and hyphens, like \`docs\`.`);
|
|
@@ -6253,13 +6938,16 @@ function readPublicationDefinition(doc) {
|
|
|
6253
6938
|
if (def.recent !== void 0) if (typeof def.recent === "number" && Number.isInteger(def.recent) && def.recent > 0) recent = def.recent;
|
|
6254
6939
|
else issue("publication-invalid-recent", "`recent` must be a whole number above zero: how many posts the front page lists (10 unless set).");
|
|
6255
6940
|
const includes = {};
|
|
6256
|
-
if (def.includes !== void 0) if (typeof def.includes === "object" && def.includes !== null && !Array.isArray(def.includes)) for (const [slot, value] of Object.entries(def.includes))
|
|
6257
|
-
|
|
6258
|
-
|
|
6259
|
-
|
|
6260
|
-
|
|
6261
|
-
|
|
6262
|
-
|
|
6941
|
+
if (def.includes !== void 0) if (typeof def.includes === "object" && def.includes !== null && !Array.isArray(def.includes)) for (const [slot, value] of Object.entries(def.includes)) {
|
|
6942
|
+
if (isEmptyValue(value)) continue;
|
|
6943
|
+
if (typeof value === "string" && value.trim() !== "") includes[slot] = value.trim();
|
|
6944
|
+
else issues.push({
|
|
6945
|
+
level: "warning",
|
|
6946
|
+
code: "publication-invalid-include",
|
|
6947
|
+
message: `Include \`${slot}\` must name the page that fills it; it is ignored.`,
|
|
6948
|
+
concept: doc.concept
|
|
6949
|
+
});
|
|
6950
|
+
}
|
|
6263
6951
|
else issue("publication-invalid-includes", "`includes` must be a mapping of include slot to the page that fills it.");
|
|
6264
6952
|
if (issues.some((i) => i.level === "error")) return {
|
|
6265
6953
|
publication: null,
|
|
@@ -6478,16 +7166,65 @@ function graphThemeFromFiles(id, name, files, origin, now = /* @__PURE__ */ new
|
|
|
6478
7166
|
*
|
|
6479
7167
|
* The transport is injected (a `connect(url)` factory) so this unit-tests without real sockets.
|
|
6480
7168
|
*/
|
|
6481
|
-
/**
|
|
6482
|
-
|
|
6483
|
-
|
|
6484
|
-
|
|
7169
|
+
/** The HTTP status a token source's refusal carries (SyncApiError, ManagedTokenError), if any. */
|
|
7170
|
+
function refusalStatus(error) {
|
|
7171
|
+
const status = typeof error === "object" && error !== null ? error.status : void 0;
|
|
7172
|
+
return typeof status === "number" ? status : void 0;
|
|
7173
|
+
}
|
|
7174
|
+
/** How long activity changes are gathered before listeners hear them: an import acks thousands. */
|
|
7175
|
+
var ACTIVITY_COALESCE_MS = 50;
|
|
6485
7176
|
/** Internal signal: a watermark request is safe to repeat on the next socket generation. */
|
|
6486
7177
|
var WatermarkConnectionInterruptedError = class extends Error {};
|
|
7178
|
+
/**
|
|
7179
|
+
* The relay did not answer in time: the socket reads open, but nothing reaches the server, as on a
|
|
7180
|
+
* connection that dropped without closing. Named so a caller can say "can't reach the Sync Server"
|
|
7181
|
+
* rather than repeat the protocol's words.
|
|
7182
|
+
*/
|
|
7183
|
+
var RelayUnansweredError = class extends Error {};
|
|
6487
7184
|
function createGraphSync(deps) {
|
|
6488
7185
|
const engines = /* @__PURE__ */ new Map();
|
|
6489
7186
|
const retained = new Map([[deps.rootDocId, 1]]);
|
|
6490
7187
|
const retiring = /* @__PURE__ */ new Set();
|
|
7188
|
+
/**
|
|
7189
|
+
* Batch walks holding an engine: one count per `seedDocsFromCache` or `readyDocs` not yet
|
|
7190
|
+
* matched by a `retireDocs`. Walks overlap - a publish's read, the Local Mirror's pass and
|
|
7191
|
+
* the index's catch-up can all be over the same document - so a count decides retirement.
|
|
7192
|
+
* With a plain flag, the first walk to finish would retire the engine while another still
|
|
7193
|
+
* waited on its catch-up; the catch-up's completion would destroy it, the waiter's
|
|
7194
|
+
* `caughtUpDoc` would resolve, and the waiter would read the empty engine created in its
|
|
7195
|
+
* place (a cold `etherpk-mcp publish` would read every page as empty and "settled").
|
|
7196
|
+
*/
|
|
7197
|
+
const batchHolds = /* @__PURE__ */ new Map();
|
|
7198
|
+
function holdForBatch(docIds) {
|
|
7199
|
+
for (const docId of docIds) {
|
|
7200
|
+
if (docId === deps.rootDocId) continue;
|
|
7201
|
+
batchHolds.set(docId, (batchHolds.get(docId) ?? 0) + 1);
|
|
7202
|
+
}
|
|
7203
|
+
}
|
|
7204
|
+
/**
|
|
7205
|
+
* Wait for a held batch's cache reads. The holds stay taken whether or not they succeed: the
|
|
7206
|
+
* caller releases the batch it asked for, once, in its own `finally`. Releasing here as well
|
|
7207
|
+
* would release a failed batch twice, and the second release would take the hold of another
|
|
7208
|
+
* walk over the same document.
|
|
7209
|
+
*/
|
|
7210
|
+
async function awaitHeld(docIds) {
|
|
7211
|
+
await Promise.all(docIds.map((docId) => readied.get(docId) ?? Promise.resolve()));
|
|
7212
|
+
}
|
|
7213
|
+
/** One hold per document back; an engine no walk holds or retains is retired once idle. */
|
|
7214
|
+
function releaseBatch(docIds) {
|
|
7215
|
+
for (const docId of docIds) {
|
|
7216
|
+
if (docId === deps.rootDocId) continue;
|
|
7217
|
+
const holds = (batchHolds.get(docId) ?? 0) - 1;
|
|
7218
|
+
if (holds > 0) {
|
|
7219
|
+
batchHolds.set(docId, holds);
|
|
7220
|
+
continue;
|
|
7221
|
+
}
|
|
7222
|
+
batchHolds.delete(docId);
|
|
7223
|
+
if (retained.has(docId)) continue;
|
|
7224
|
+
retiring.add(docId);
|
|
7225
|
+
retireEngineIfIdle(docId);
|
|
7226
|
+
}
|
|
7227
|
+
}
|
|
6491
7228
|
const presenceSession = deps.presence ? createPresenceSession({
|
|
6492
7229
|
name: deps.presence.name,
|
|
6493
7230
|
color: deps.presence.color,
|
|
@@ -6499,6 +7236,42 @@ function createGraphSync(deps) {
|
|
|
6499
7236
|
let socket;
|
|
6500
7237
|
let open = false;
|
|
6501
7238
|
let disposed = false;
|
|
7239
|
+
/** Set once access has ended for good; nothing reconnects after it. */
|
|
7240
|
+
let lost = null;
|
|
7241
|
+
/** See `SyncActivity.connection`. */
|
|
7242
|
+
let connection = "connecting";
|
|
7243
|
+
/** Documents whose engines hold unacknowledged outbox operations, kept by `onQueueChange`. */
|
|
7244
|
+
const unsentDocs = /* @__PURE__ */ new Set();
|
|
7245
|
+
/**
|
|
7246
|
+
* Write refusals. The relay answers a refused append, delete or resurrect with `quota_denied`
|
|
7247
|
+
* naming the document and operation, and acknowledges nothing: the operation stays at the
|
|
7248
|
+
* head of that document's outbox, and without a retry the document would stay stalled until
|
|
7249
|
+
* a reload even after the allowance came back. `refusedDocs` are the documents to send
|
|
7250
|
+
* again; empty when a server too old to name them refused.
|
|
7251
|
+
*/
|
|
7252
|
+
let refusal = null;
|
|
7253
|
+
const refusedDocs = /* @__PURE__ */ new Set();
|
|
7254
|
+
/** Bumped by every refusal, so `awaitAcked` can tell one of its own writes from an older one. */
|
|
7255
|
+
let refusalCount = 0;
|
|
7256
|
+
let refusalRounds = 0;
|
|
7257
|
+
let refusalTimer;
|
|
7258
|
+
const refusalDelay = deps.refusalRetryDelayMs ?? ((round) => refusalRetryDelayMs(round));
|
|
7259
|
+
const activityListeners = /* @__PURE__ */ new Set();
|
|
7260
|
+
let activityTimer;
|
|
7261
|
+
const snapshotActivity = () => ({
|
|
7262
|
+
connection,
|
|
7263
|
+
unsentDocuments: unsentDocs.size,
|
|
7264
|
+
refusal
|
|
7265
|
+
});
|
|
7266
|
+
function activityChanged() {
|
|
7267
|
+
if (disposed || activityTimer || activityListeners.size === 0) return;
|
|
7268
|
+
activityTimer = setTimeout(() => {
|
|
7269
|
+
activityTimer = void 0;
|
|
7270
|
+
if (disposed) return;
|
|
7271
|
+
const current = snapshotActivity();
|
|
7272
|
+
for (const listener of activityListeners) listener(current);
|
|
7273
|
+
}, ACTIVITY_COALESCE_MS);
|
|
7274
|
+
}
|
|
6502
7275
|
/**
|
|
6503
7276
|
* Set when the Sync Server turned out to speak another protocol version. The session then
|
|
6504
7277
|
* stops for good: every reconnect would meet the same server, and nothing either side
|
|
@@ -6509,6 +7282,10 @@ function createGraphSync(deps) {
|
|
|
6509
7282
|
if (!disposed) deps.onError?.(error instanceof Error ? error : new Error(String(error)));
|
|
6510
7283
|
};
|
|
6511
7284
|
const outboundSnapshots = [];
|
|
7285
|
+
/** Documents whose operation this socket has sent and the relay has not answered, to its outbox id. */
|
|
7286
|
+
const operationsInFlight = /* @__PURE__ */ new Map();
|
|
7287
|
+
/** Each document's operation waiting for room in {@link OPERATION_WINDOW}, oldest first. */
|
|
7288
|
+
const operationsWaiting = /* @__PURE__ */ new Map();
|
|
6512
7289
|
const foregroundCatchups = [];
|
|
6513
7290
|
const backgroundCatchups = [];
|
|
6514
7291
|
let catchupInFlight;
|
|
@@ -6526,6 +7303,7 @@ function createGraphSync(deps) {
|
|
|
6526
7303
|
function waitForCurrentConnection() {
|
|
6527
7304
|
if (open && socket) return Promise.resolve();
|
|
6528
7305
|
if (disposed) return Promise.reject(/* @__PURE__ */ new Error("the graph sync session was disposed"));
|
|
7306
|
+
if (lost) return Promise.reject(/* @__PURE__ */ new Error("access to this graph has ended"));
|
|
6529
7307
|
if (protocolMismatch) return Promise.reject(protocolMismatch);
|
|
6530
7308
|
return new Promise((resolve, reject) => {
|
|
6531
7309
|
connectionWaiters.add({
|
|
@@ -6544,7 +7322,7 @@ function createGraphSync(deps) {
|
|
|
6544
7322
|
const requestId = crypto.randomUUID();
|
|
6545
7323
|
const timer = setTimeout(() => {
|
|
6546
7324
|
watermarkRequests.delete(requestId);
|
|
6547
|
-
reject(
|
|
7325
|
+
reject(new RelayUnansweredError("the sync relay did not answer a watermark check"));
|
|
6548
7326
|
}, 15e3);
|
|
6549
7327
|
watermarkRequests.set(requestId, {
|
|
6550
7328
|
resolve,
|
|
@@ -6558,7 +7336,7 @@ function createGraphSync(deps) {
|
|
|
6558
7336
|
}));
|
|
6559
7337
|
});
|
|
6560
7338
|
} catch (error) {
|
|
6561
|
-
if (error instanceof WatermarkConnectionInterruptedError && !disposed) continue;
|
|
7339
|
+
if (error instanceof WatermarkConnectionInterruptedError && !disposed && !lost) continue;
|
|
6562
7340
|
throw error;
|
|
6563
7341
|
}
|
|
6564
7342
|
}
|
|
@@ -6620,6 +7398,37 @@ function createGraphSync(deps) {
|
|
|
6620
7398
|
pumpCatchups();
|
|
6621
7399
|
retireEngineIfIdle(completedDocId);
|
|
6622
7400
|
}
|
|
7401
|
+
/**
|
|
7402
|
+
* Send a document's operation within the window, or hold it until an answer makes room. A
|
|
7403
|
+
* document's newer operation replaces one still waiting (the engine sends only its head).
|
|
7404
|
+
*/
|
|
7405
|
+
function sendOperation(docId, message) {
|
|
7406
|
+
ackRoute.set(message.outboxId, docId);
|
|
7407
|
+
if (!open || !socket) return;
|
|
7408
|
+
if (!operationsInFlight.has(docId) && operationsInFlight.size >= 128) {
|
|
7409
|
+
operationsWaiting.set(docId, message);
|
|
7410
|
+
return;
|
|
7411
|
+
}
|
|
7412
|
+
operationsWaiting.delete(docId);
|
|
7413
|
+
operationsInFlight.set(docId, message.outboxId);
|
|
7414
|
+
sendNow(message);
|
|
7415
|
+
}
|
|
7416
|
+
/**
|
|
7417
|
+
* The relay answered `docId`'s operation, by acking or refusing it: its place in the window
|
|
7418
|
+
* goes to the longest-waiting one. `outboxId`, when the answer names one, must be the
|
|
7419
|
+
* operation in flight; an answer to an older one frees nothing.
|
|
7420
|
+
*/
|
|
7421
|
+
function operationAnswered(docId, outboxId) {
|
|
7422
|
+
const inFlight = operationsInFlight.get(docId);
|
|
7423
|
+
if (inFlight === void 0 || outboxId !== void 0 && inFlight !== outboxId) return;
|
|
7424
|
+
operationsInFlight.delete(docId);
|
|
7425
|
+
for (const [waitingDocId, waiting] of operationsWaiting) {
|
|
7426
|
+
if (!open || !socket || operationsInFlight.size >= 128) return;
|
|
7427
|
+
operationsWaiting.delete(waitingDocId);
|
|
7428
|
+
operationsInFlight.set(waitingDocId, waiting.outboxId);
|
|
7429
|
+
sendNow(waiting);
|
|
7430
|
+
}
|
|
7431
|
+
}
|
|
6623
7432
|
function rawSend(message) {
|
|
6624
7433
|
if (message.type === "catchup") {
|
|
6625
7434
|
queueCatchup(message);
|
|
@@ -6655,7 +7464,7 @@ function createGraphSync(deps) {
|
|
|
6655
7464
|
presenceDetach.delete(docId);
|
|
6656
7465
|
}
|
|
6657
7466
|
function retireEngineIfIdle(docId) {
|
|
6658
|
-
if (docId === deps.rootDocId || retained.has(docId) || !retiring.has(docId)) return;
|
|
7467
|
+
if (docId === deps.rootDocId || retained.has(docId) || batchHolds.has(docId) || !retiring.has(docId)) return;
|
|
6659
7468
|
if (catchupInFlight?.docId === docId || foregroundCatchups.some((request) => request.docId === docId) || backgroundCatchups.some((request) => request.docId === docId)) return;
|
|
6660
7469
|
const candidate = engines.get(docId);
|
|
6661
7470
|
if (!candidate || !candidate.isIdle()) return;
|
|
@@ -6663,6 +7472,7 @@ function createGraphSync(deps) {
|
|
|
6663
7472
|
detachPresence(docId);
|
|
6664
7473
|
candidate.destroy();
|
|
6665
7474
|
engines.delete(docId);
|
|
7475
|
+
unsentDocs.delete(docId);
|
|
6666
7476
|
readied.delete(docId);
|
|
6667
7477
|
syncEnabled.delete(docId);
|
|
6668
7478
|
performanceRecorder.mark("sync.engine.count", {
|
|
@@ -6670,6 +7480,69 @@ function createGraphSync(deps) {
|
|
|
6670
7480
|
subscriptions: retained.size
|
|
6671
7481
|
});
|
|
6672
7482
|
}
|
|
7483
|
+
function noteQueue(docId) {
|
|
7484
|
+
const e = engines.get(docId);
|
|
7485
|
+
if (e && e.unsentOperations() > 0) unsentDocs.add(docId);
|
|
7486
|
+
else unsentDocs.delete(docId);
|
|
7487
|
+
activityChanged();
|
|
7488
|
+
}
|
|
7489
|
+
/**
|
|
7490
|
+
* The relay refused a write on a quota. A refused snapshot is not a stalled write - the relay
|
|
7491
|
+
* still holds the document's log, and the engine uploads another at a later idle - so only a
|
|
7492
|
+
* refused outbox operation (named by its outbox id), or a refusal a server too old to name
|
|
7493
|
+
* anything sent, counts.
|
|
7494
|
+
*/
|
|
7495
|
+
function noteRefusal(message) {
|
|
7496
|
+
if (message.docId && !message.outboxId) return;
|
|
7497
|
+
if (message.docId) refusedDocs.add(message.docId);
|
|
7498
|
+
refusal = message.quotaCode ? { quotaCode: message.quotaCode } : {};
|
|
7499
|
+
refusalCount += 1;
|
|
7500
|
+
scheduleRefusalRetry();
|
|
7501
|
+
activityChanged();
|
|
7502
|
+
}
|
|
7503
|
+
function scheduleRefusalRetry() {
|
|
7504
|
+
if (refusalTimer || disposed || lost) return;
|
|
7505
|
+
refusalTimer = setTimeout(() => {
|
|
7506
|
+
refusalTimer = void 0;
|
|
7507
|
+
refusalRounds += 1;
|
|
7508
|
+
resendRefused();
|
|
7509
|
+
}, refusalDelay(refusalRounds + 1));
|
|
7510
|
+
}
|
|
7511
|
+
/** Send each refused document's head operation again; the relay applies an outbox id once. */
|
|
7512
|
+
function resendRefused() {
|
|
7513
|
+
if (disposed || lost || !open) return;
|
|
7514
|
+
const docIds = refusedDocs.size > 0 ? [...refusedDocs] : [...unsentDocs];
|
|
7515
|
+
for (const docId of docIds) {
|
|
7516
|
+
const e = engines.get(docId);
|
|
7517
|
+
if (!e || e.unsentOperations() === 0) {
|
|
7518
|
+
refusedDocs.delete(docId);
|
|
7519
|
+
continue;
|
|
7520
|
+
}
|
|
7521
|
+
e.retryUnsent();
|
|
7522
|
+
}
|
|
7523
|
+
if (refusedDocs.size === 0 && unsentDocs.size === 0) clearRefusal();
|
|
7524
|
+
}
|
|
7525
|
+
/** Whether every document with unsent operations is one the server refused. */
|
|
7526
|
+
function everyUnsentRefused() {
|
|
7527
|
+
if (unsentDocs.size === 0) return false;
|
|
7528
|
+
if (refusedDocs.size === 0) return true;
|
|
7529
|
+
for (const docId of unsentDocs) if (!refusedDocs.has(docId)) return false;
|
|
7530
|
+
return true;
|
|
7531
|
+
}
|
|
7532
|
+
/** An ack: the server accepted a write, so that document is refused no longer. */
|
|
7533
|
+
function writeAccepted(docId) {
|
|
7534
|
+
if (!refusal) return;
|
|
7535
|
+
refusedDocs.delete(docId);
|
|
7536
|
+
if (refusedDocs.size === 0) clearRefusal();
|
|
7537
|
+
}
|
|
7538
|
+
function clearRefusal() {
|
|
7539
|
+
refusal = null;
|
|
7540
|
+
refusedDocs.clear();
|
|
7541
|
+
refusalRounds = 0;
|
|
7542
|
+
clearTimeout(refusalTimer);
|
|
7543
|
+
refusalTimer = void 0;
|
|
7544
|
+
activityChanged();
|
|
7545
|
+
}
|
|
6673
7546
|
function engine(docId, synchronize = true) {
|
|
6674
7547
|
let e = engines.get(docId);
|
|
6675
7548
|
if (!e) {
|
|
@@ -6678,7 +7551,10 @@ function createGraphSync(deps) {
|
|
|
6678
7551
|
graphId: deps.graphId,
|
|
6679
7552
|
keyring: deps.keyring,
|
|
6680
7553
|
send: (message) => {
|
|
6681
|
-
if (message.type === "append" || message.type === "delete" || message.type === "resurrect")
|
|
7554
|
+
if (message.type === "append" || message.type === "delete" || message.type === "resurrect") {
|
|
7555
|
+
sendOperation(docId, message);
|
|
7556
|
+
return;
|
|
7557
|
+
}
|
|
6682
7558
|
rawSend(message);
|
|
6683
7559
|
},
|
|
6684
7560
|
persist: deps.cache.docCache(docId, docId === deps.rootDocId ? "root" : "document"),
|
|
@@ -6686,6 +7562,7 @@ function createGraphSync(deps) {
|
|
|
6686
7562
|
collectRowWhen: docId === deps.rootDocId ? void 0 : deps.collectRowWhen,
|
|
6687
7563
|
onError: reportError,
|
|
6688
7564
|
onIdle: () => queueMicrotask(() => retireEngineIfIdle(docId)),
|
|
7565
|
+
onQueueChange: () => noteQueue(docId),
|
|
6689
7566
|
catchupPriority: () => {
|
|
6690
7567
|
return retained.has(docId) ? "foreground" : "background";
|
|
6691
7568
|
}
|
|
@@ -6782,6 +7659,11 @@ function createGraphSync(deps) {
|
|
|
6782
7659
|
function stopForProtocolMismatch(serverVersion) {
|
|
6783
7660
|
if (protocolMismatch) return;
|
|
6784
7661
|
protocolMismatch = new SyncProtocolMismatchError(serverVersion);
|
|
7662
|
+
connection = "ended";
|
|
7663
|
+
clearTimeout(reconnectTimer);
|
|
7664
|
+
clearTimeout(refusalTimer);
|
|
7665
|
+
refusalTimer = void 0;
|
|
7666
|
+
activityChanged();
|
|
6785
7667
|
reportError(protocolMismatch);
|
|
6786
7668
|
for (const waiter of connectionWaiters) waiter.reject(protocolMismatch);
|
|
6787
7669
|
connectionWaiters.clear();
|
|
@@ -6801,8 +7683,18 @@ function createGraphSync(deps) {
|
|
|
6801
7683
|
continuation: message.hasMore ? 1 : 0
|
|
6802
7684
|
});
|
|
6803
7685
|
if (message.type === "error") {
|
|
7686
|
+
if (message.code === "membership_revoked") {
|
|
7687
|
+
endAccess({ kind: "membership" });
|
|
7688
|
+
return;
|
|
7689
|
+
}
|
|
7690
|
+
if (message.code === "quota_denied") {
|
|
7691
|
+
if (message.docId) operationAnswered(message.docId, message.outboxId);
|
|
7692
|
+
noteRefusal(message);
|
|
7693
|
+
return;
|
|
7694
|
+
}
|
|
6804
7695
|
if (message.code === "stale_generation" && message.docId && message.currentGeneration !== void 0) {
|
|
6805
7696
|
if (catchupInFlight?.docId === message.docId) catchupInFlight = void 0;
|
|
7697
|
+
operationAnswered(message.docId);
|
|
6806
7698
|
engine(message.docId).staleGeneration(message.currentGeneration);
|
|
6807
7699
|
pumpCatchups();
|
|
6808
7700
|
}
|
|
@@ -6824,7 +7716,13 @@ function createGraphSync(deps) {
|
|
|
6824
7716
|
}
|
|
6825
7717
|
if (message.type === "ack") {
|
|
6826
7718
|
const docId = ackRoute.get(message.outboxId);
|
|
6827
|
-
if (docId)
|
|
7719
|
+
if (docId) {
|
|
7720
|
+
operationAnswered(docId, message.outboxId);
|
|
7721
|
+
engine(docId).receive(message).then(() => {
|
|
7722
|
+
ackRoute.delete(message.outboxId);
|
|
7723
|
+
writeAccepted(docId);
|
|
7724
|
+
}).catch(report);
|
|
7725
|
+
}
|
|
6828
7726
|
return;
|
|
6829
7727
|
}
|
|
6830
7728
|
if (message.type === "catchup_batch") {
|
|
@@ -6834,20 +7732,78 @@ function createGraphSync(deps) {
|
|
|
6834
7732
|
engine(message.docId).receive(message).catch(report);
|
|
6835
7733
|
}
|
|
6836
7734
|
/**
|
|
7735
|
+
* Forget everything that belonged to one socket generation. Durable IndexedDB rows, not these
|
|
7736
|
+
* entries, determine outstanding work and replay on the next open, which resubscribes every
|
|
7737
|
+
* retained document.
|
|
7738
|
+
*/
|
|
7739
|
+
function forgetSocketGeneration() {
|
|
7740
|
+
subscribed.clear();
|
|
7741
|
+
ackRoute.clear();
|
|
7742
|
+
operationsInFlight.clear();
|
|
7743
|
+
operationsWaiting.clear();
|
|
7744
|
+
catchupInFlight = void 0;
|
|
7745
|
+
foregroundCatchups.length = 0;
|
|
7746
|
+
backgroundCatchups.length = 0;
|
|
7747
|
+
for (const pending of watermarkRequests.values()) {
|
|
7748
|
+
clearTimeout(pending.timer);
|
|
7749
|
+
pending.reject(new WatermarkConnectionInterruptedError());
|
|
7750
|
+
}
|
|
7751
|
+
watermarkRequests.clear();
|
|
7752
|
+
}
|
|
7753
|
+
let attempts = 0;
|
|
7754
|
+
let openedAt = 0;
|
|
7755
|
+
let forceToken = false;
|
|
7756
|
+
let credentialRefusals = 0;
|
|
7757
|
+
let reconnectTimer;
|
|
7758
|
+
const retryDelay = deps.retryDelayMs ?? ((attempt) => reconnectDelayMs(attempt));
|
|
7759
|
+
function scheduleConnect(delayMs) {
|
|
7760
|
+
if (disposed || lost || protocolMismatch) return;
|
|
7761
|
+
clearTimeout(reconnectTimer);
|
|
7762
|
+
reconnectTimer = setTimeout(connect, delayMs);
|
|
7763
|
+
}
|
|
7764
|
+
function backOff() {
|
|
7765
|
+
attempts += 1;
|
|
7766
|
+
scheduleConnect(retryDelay(attempts));
|
|
7767
|
+
}
|
|
7768
|
+
/** Stop for good: close the socket, never reconnect, and tell the owner once. */
|
|
7769
|
+
function endAccess(loss) {
|
|
7770
|
+
if (lost || disposed) return;
|
|
7771
|
+
lost = loss;
|
|
7772
|
+
connection = "ended";
|
|
7773
|
+
clearTimeout(reconnectTimer);
|
|
7774
|
+
clearTimeout(refusalTimer);
|
|
7775
|
+
refusalTimer = void 0;
|
|
7776
|
+
activityChanged();
|
|
7777
|
+
const current = socket;
|
|
7778
|
+
socket = void 0;
|
|
7779
|
+
open = false;
|
|
7780
|
+
forgetSocketGeneration();
|
|
7781
|
+
current?.close();
|
|
7782
|
+
for (const waiter of connectionWaiters) waiter.reject(/* @__PURE__ */ new Error("access to this graph has ended"));
|
|
7783
|
+
connectionWaiters.clear();
|
|
7784
|
+
deps.onAccessLost?.(loss);
|
|
7785
|
+
}
|
|
7786
|
+
/**
|
|
6837
7787
|
* A token is fetched for EVERY connect, never captured: the source re-mints when the
|
|
6838
|
-
* held token nears expiry, so a socket that drops an hour in still reconnects.
|
|
7788
|
+
* held token nears expiry, so a socket that drops an hour in still reconnects.
|
|
6839
7789
|
* Encrypted appends issued while the token is in flight remain in IndexedDB and replay
|
|
6840
7790
|
* on open. Only rebuildable snapshot uploads use a volatile queue.
|
|
6841
7791
|
*/
|
|
6842
7792
|
function connect() {
|
|
6843
|
-
if (disposed || protocolMismatch) return;
|
|
6844
|
-
|
|
6845
|
-
|
|
7793
|
+
if (disposed || lost || protocolMismatch) return;
|
|
7794
|
+
const force = forceToken;
|
|
7795
|
+
forceToken = false;
|
|
7796
|
+
deps.token(force ? { force: true } : void 0).then((token) => {
|
|
7797
|
+
if (disposed || lost) return;
|
|
6846
7798
|
const s = deps.connect(`${deps.relayUrl}?token=${encodeURIComponent(token)}`);
|
|
6847
7799
|
socket = s;
|
|
6848
7800
|
s.onOpen(() => {
|
|
6849
|
-
if (disposed || socket !== s) return;
|
|
7801
|
+
if (disposed || lost || socket !== s) return;
|
|
6850
7802
|
open = true;
|
|
7803
|
+
connection = "open";
|
|
7804
|
+
activityChanged();
|
|
7805
|
+
openedAt = Date.now();
|
|
7806
|
+
credentialRefusals = 0;
|
|
6851
7807
|
resolveFirstOpen?.();
|
|
6852
7808
|
for (const waiter of connectionWaiters) waiter.resolve();
|
|
6853
7809
|
connectionWaiters.clear();
|
|
@@ -6868,24 +7824,45 @@ function createGraphSync(deps) {
|
|
|
6868
7824
|
}
|
|
6869
7825
|
});
|
|
6870
7826
|
s.onMessage(handleMessage);
|
|
6871
|
-
s.onClose(() => {
|
|
7827
|
+
s.onClose((event) => {
|
|
6872
7828
|
if (socket !== s) return;
|
|
7829
|
+
const upFor = open ? Date.now() - openedAt : 0;
|
|
6873
7830
|
open = false;
|
|
6874
7831
|
socket = void 0;
|
|
6875
|
-
|
|
6876
|
-
|
|
6877
|
-
|
|
6878
|
-
|
|
6879
|
-
|
|
6880
|
-
|
|
6881
|
-
|
|
6882
|
-
|
|
7832
|
+
forgetSocketGeneration();
|
|
7833
|
+
if (disposed || lost || protocolMismatch) return;
|
|
7834
|
+
connection = "reconnecting";
|
|
7835
|
+
activityChanged();
|
|
7836
|
+
if (event?.code === 4403) {
|
|
7837
|
+
endAccess({ kind: "membership" });
|
|
7838
|
+
return;
|
|
7839
|
+
}
|
|
7840
|
+
if (upFor >= 1e4) attempts = 0;
|
|
7841
|
+
if (event?.code === 4401) {
|
|
7842
|
+
forceToken = true;
|
|
7843
|
+
credentialRefusals += 1;
|
|
7844
|
+
if (credentialRefusals === 1) {
|
|
7845
|
+
scheduleConnect(0);
|
|
7846
|
+
return;
|
|
7847
|
+
}
|
|
6883
7848
|
}
|
|
6884
|
-
|
|
6885
|
-
if (!disposed && !protocolMismatch) setTimeout(connect, RECONNECT_MS);
|
|
7849
|
+
backOff();
|
|
6886
7850
|
});
|
|
6887
|
-
}, () => {
|
|
6888
|
-
if (
|
|
7851
|
+
}, (error) => {
|
|
7852
|
+
if (disposed || lost) return;
|
|
7853
|
+
const status = refusalStatus(error);
|
|
7854
|
+
if (status === 401) {
|
|
7855
|
+
endAccess({
|
|
7856
|
+
kind: "credentials",
|
|
7857
|
+
cause: error
|
|
7858
|
+
});
|
|
7859
|
+
return;
|
|
7860
|
+
}
|
|
7861
|
+
if (status === 403) {
|
|
7862
|
+
endAccess({ kind: "membership" });
|
|
7863
|
+
return;
|
|
7864
|
+
}
|
|
7865
|
+
backOff();
|
|
6889
7866
|
});
|
|
6890
7867
|
}
|
|
6891
7868
|
connect();
|
|
@@ -7064,21 +8041,17 @@ function createGraphSync(deps) {
|
|
|
7064
8041
|
await readied.get(docId);
|
|
7065
8042
|
},
|
|
7066
8043
|
async seedDocsFromCache(docIds) {
|
|
8044
|
+
holdForBatch(docIds);
|
|
7067
8045
|
const batch = docIds.map((docId) => engine(docId, false));
|
|
7068
|
-
await
|
|
8046
|
+
await awaitHeld(docIds);
|
|
7069
8047
|
return batch.map((entry) => entry.doc);
|
|
7070
8048
|
},
|
|
7071
8049
|
async readyDocs(docIds) {
|
|
8050
|
+
holdForBatch(docIds);
|
|
7072
8051
|
for (const docId of docIds) engine(docId);
|
|
7073
|
-
await
|
|
7074
|
-
},
|
|
7075
|
-
retireDocs(docIds) {
|
|
7076
|
-
for (const docId of docIds) {
|
|
7077
|
-
if (docId === deps.rootDocId || retained.has(docId)) continue;
|
|
7078
|
-
retiring.add(docId);
|
|
7079
|
-
retireEngineIfIdle(docId);
|
|
7080
|
-
}
|
|
8052
|
+
await awaitHeld(docIds);
|
|
7081
8053
|
},
|
|
8054
|
+
retireDocs: releaseBatch,
|
|
7082
8055
|
firstCatchupPageDoc: (docId) => engine(docId).firstCatchupPage(),
|
|
7083
8056
|
caughtUpDoc: (docId) => engine(docId).caughtUp(),
|
|
7084
8057
|
async docsNeedingCatchup(docIds) {
|
|
@@ -7105,12 +8078,24 @@ function createGraphSync(deps) {
|
|
|
7105
8078
|
};
|
|
7106
8079
|
},
|
|
7107
8080
|
isConnected: () => open,
|
|
8081
|
+
activity: snapshotActivity,
|
|
8082
|
+
onActivity(listener) {
|
|
8083
|
+
activityListeners.add(listener);
|
|
8084
|
+
return () => activityListeners.delete(listener);
|
|
8085
|
+
},
|
|
8086
|
+
retryRefused() {
|
|
8087
|
+
if (!refusal) return;
|
|
8088
|
+
clearTimeout(refusalTimer);
|
|
8089
|
+
refusalTimer = void 0;
|
|
8090
|
+
resendRefused();
|
|
8091
|
+
},
|
|
7108
8092
|
async flushAll({ onProgress } = {}) {
|
|
7109
8093
|
const all = [...engines.values()];
|
|
7110
8094
|
let flushed = 0;
|
|
7111
8095
|
await Promise.all(all.map((e) => e.flush().then(() => onProgress?.(++flushed, all.length))));
|
|
7112
8096
|
},
|
|
7113
8097
|
async awaitAcked({ onProgress, signal, stallMs = 3e4 } = {}) {
|
|
8098
|
+
const refusalsBefore = refusalCount;
|
|
7114
8099
|
const initial = await deps.cache.countPending();
|
|
7115
8100
|
if (initial === 0) return {
|
|
7116
8101
|
settled: true,
|
|
@@ -7138,13 +8123,19 @@ function createGraphSync(deps) {
|
|
|
7138
8123
|
settled: true,
|
|
7139
8124
|
outstanding: 0
|
|
7140
8125
|
});
|
|
8126
|
+
if (refusal && (refusalCount > refusalsBefore || everyUnsentRefused())) return finish({
|
|
8127
|
+
settled: false,
|
|
8128
|
+
outstanding: current,
|
|
8129
|
+
refused: refusal
|
|
8130
|
+
});
|
|
7141
8131
|
if (signal?.aborted) return finish({
|
|
7142
8132
|
settled: false,
|
|
7143
8133
|
outstanding: current
|
|
7144
8134
|
});
|
|
7145
8135
|
if (Date.now() - lastProgressAt >= stallMs) return finish({
|
|
7146
8136
|
settled: false,
|
|
7147
|
-
outstanding: current
|
|
8137
|
+
outstanding: current,
|
|
8138
|
+
...refusal ? { refused: refusal } : {}
|
|
7148
8139
|
});
|
|
7149
8140
|
timer = setTimeout(poll, POLL_MS);
|
|
7150
8141
|
} catch (error) {
|
|
@@ -7162,14 +8153,21 @@ function createGraphSync(deps) {
|
|
|
7162
8153
|
activeEngines: engines.size,
|
|
7163
8154
|
retainedDocuments: retained.size
|
|
7164
8155
|
}),
|
|
8156
|
+
endAccess,
|
|
8157
|
+
accessLoss: () => lost,
|
|
7165
8158
|
dispose() {
|
|
7166
8159
|
disposed = true;
|
|
8160
|
+
clearTimeout(reconnectTimer);
|
|
8161
|
+
clearTimeout(refusalTimer);
|
|
8162
|
+
clearTimeout(activityTimer);
|
|
8163
|
+
activityListeners.clear();
|
|
7167
8164
|
socket?.close();
|
|
7168
8165
|
for (const detach of presenceDetach.values()) detach();
|
|
7169
8166
|
presenceDetach.clear();
|
|
7170
8167
|
for (const e of engines.values()) e.destroy();
|
|
7171
8168
|
engines.clear();
|
|
7172
8169
|
retiring.clear();
|
|
8170
|
+
batchHolds.clear();
|
|
7173
8171
|
syncEnabled.clear();
|
|
7174
8172
|
foregroundCatchups.length = 0;
|
|
7175
8173
|
backgroundCatchups.length = 0;
|
|
@@ -7649,7 +8647,10 @@ function nodeTransport(url) {
|
|
|
7649
8647
|
close: () => ws.close(),
|
|
7650
8648
|
onOpen: (cb) => ws.addEventListener("open", () => cb()),
|
|
7651
8649
|
onMessage: (cb) => ws.addEventListener("message", (e) => cb(String(e.data))),
|
|
7652
|
-
onClose: (cb) => ws.addEventListener("close", () => cb(
|
|
8650
|
+
onClose: (cb) => ws.addEventListener("close", (event) => cb({
|
|
8651
|
+
code: event.code,
|
|
8652
|
+
reason: event.reason
|
|
8653
|
+
}))
|
|
7653
8654
|
};
|
|
7654
8655
|
}
|
|
7655
8656
|
//#endregion
|
|
@@ -8011,7 +9012,7 @@ function createIndexCore(host) {
|
|
|
8011
9012
|
}];
|
|
8012
9013
|
}
|
|
8013
9014
|
case "search-text": {
|
|
8014
|
-
const page = db ? searchText$1(db, request.query, request.offset, request.limit) : {
|
|
9015
|
+
const page = db ? searchText$1(db, request.query, request.offset, request.limit, request.filters ?? []) : {
|
|
8015
9016
|
groups: [],
|
|
8016
9017
|
hasMore: false
|
|
8017
9018
|
};
|
|
@@ -8023,7 +9024,7 @@ function createIndexCore(host) {
|
|
|
8023
9024
|
}];
|
|
8024
9025
|
}
|
|
8025
9026
|
case "search-count": {
|
|
8026
|
-
const count = db ? searchTextCount(db, request.query) : {
|
|
9027
|
+
const count = db ? searchTextCount(db, request.query, request.filters ?? []) : {
|
|
8027
9028
|
total: 0,
|
|
8028
9029
|
capped: false
|
|
8029
9030
|
};
|
|
@@ -8034,6 +9035,21 @@ function createIndexCore(host) {
|
|
|
8034
9035
|
capped: count.capped
|
|
8035
9036
|
}];
|
|
8036
9037
|
}
|
|
9038
|
+
case "property-keys": return [{
|
|
9039
|
+
type: "property-keys",
|
|
9040
|
+
id: request.id,
|
|
9041
|
+
keys: db ? propertyKeys(db) : []
|
|
9042
|
+
}];
|
|
9043
|
+
case "property-values": return [{
|
|
9044
|
+
type: "property-values",
|
|
9045
|
+
id: request.id,
|
|
9046
|
+
values: db ? propertyValues(db, request.key) : []
|
|
9047
|
+
}];
|
|
9048
|
+
case "property-match": return [{
|
|
9049
|
+
type: "property-match",
|
|
9050
|
+
id: request.id,
|
|
9051
|
+
documents: db ? documentsMatchingProperties(db, request.filters) : []
|
|
9052
|
+
}];
|
|
8037
9053
|
case "tasks": {
|
|
8038
9054
|
const page = db ? tasksMatching(db, request.query, request.offset, request.limit) : {
|
|
8039
9055
|
hits: [],
|
|
@@ -8751,29 +9767,51 @@ function createRemoteGraphIndex(source, transport, options) {
|
|
|
8751
9767
|
needles: [...needles]
|
|
8752
9768
|
}))).usage;
|
|
8753
9769
|
},
|
|
8754
|
-
async searchText(query, offset, limit) {
|
|
9770
|
+
async searchText(query, offset, limit, filters) {
|
|
8755
9771
|
const response = await request((id) => ({
|
|
8756
9772
|
type: "search-text",
|
|
8757
9773
|
id,
|
|
8758
9774
|
query,
|
|
8759
9775
|
offset,
|
|
8760
|
-
limit
|
|
9776
|
+
limit,
|
|
9777
|
+
...filters?.length ? { filters: [...filters] } : {}
|
|
8761
9778
|
}));
|
|
8762
9779
|
return {
|
|
8763
9780
|
groups: response.groups,
|
|
8764
9781
|
hasMore: response.hasMore
|
|
8765
9782
|
};
|
|
8766
9783
|
},
|
|
8767
|
-
async searchTextCount(query) {
|
|
9784
|
+
async searchTextCount(query, filters) {
|
|
8768
9785
|
const response = await request((id) => ({
|
|
8769
9786
|
type: "search-count",
|
|
8770
9787
|
id,
|
|
8771
|
-
query
|
|
8772
|
-
|
|
8773
|
-
|
|
8774
|
-
|
|
8775
|
-
|
|
8776
|
-
|
|
9788
|
+
query,
|
|
9789
|
+
...filters?.length ? { filters: [...filters] } : {}
|
|
9790
|
+
}));
|
|
9791
|
+
return {
|
|
9792
|
+
total: response.total,
|
|
9793
|
+
capped: response.capped
|
|
9794
|
+
};
|
|
9795
|
+
},
|
|
9796
|
+
async propertyKeys() {
|
|
9797
|
+
return (await request((id) => ({
|
|
9798
|
+
type: "property-keys",
|
|
9799
|
+
id
|
|
9800
|
+
}))).keys;
|
|
9801
|
+
},
|
|
9802
|
+
async propertyValues(key) {
|
|
9803
|
+
return (await request((id) => ({
|
|
9804
|
+
type: "property-values",
|
|
9805
|
+
id,
|
|
9806
|
+
key
|
|
9807
|
+
}))).values;
|
|
9808
|
+
},
|
|
9809
|
+
async propertyMatch(filters) {
|
|
9810
|
+
return (await request((id) => ({
|
|
9811
|
+
type: "property-match",
|
|
9812
|
+
id,
|
|
9813
|
+
filters: [...filters]
|
|
9814
|
+
}))).documents;
|
|
8777
9815
|
},
|
|
8778
9816
|
async tasks(query, offset, limit) {
|
|
8779
9817
|
const response = await request((id) => ({
|
|
@@ -9065,6 +10103,20 @@ var IMAGE_EXTS = new Set([
|
|
|
9065
10103
|
]);
|
|
9066
10104
|
/** A doc-relative asset reference: optional `../` hops, then `assets/<name>`. */
|
|
9067
10105
|
var ASSET_REF$1 = /^(?:\.\.\/)*assets\/(.+)$/;
|
|
10106
|
+
/**
|
|
10107
|
+
* An asset could not be fetched just now: the connection failed, or the server answered with a
|
|
10108
|
+
* status that means "try again". It says nothing about whether the file exists, so something
|
|
10109
|
+
* showing the file asks again rather than calling it missing. `status` is the server's answer,
|
|
10110
|
+
* absent when the connection failed before there was one.
|
|
10111
|
+
*/
|
|
10112
|
+
var AssetUnavailableError = class extends Error {
|
|
10113
|
+
name = "AssetUnavailableError";
|
|
10114
|
+
status;
|
|
10115
|
+
constructor(message, options = {}) {
|
|
10116
|
+
super(message, options.cause === void 0 ? void 0 : { cause: options.cause });
|
|
10117
|
+
this.status = options.status;
|
|
10118
|
+
}
|
|
10119
|
+
};
|
|
9068
10120
|
/** Split a file name into its stem and lower-cased extension (`''` ext when there is none). */
|
|
9069
10121
|
function splitNameExt(fileName) {
|
|
9070
10122
|
const dot = fileName.lastIndexOf(".");
|
|
@@ -9504,6 +10556,25 @@ async function deleteGraphTheme(adapter, id) {
|
|
|
9504
10556
|
* Pure: no store, no editor. The YAML rules (what counts as a block, how it parses) come from
|
|
9505
10557
|
* `storage/fs`, shared with the scan and the editor's analysis, so there is one rule.
|
|
9506
10558
|
*/
|
|
10559
|
+
/** What the block claims. An unterminated block is not a block, as everywhere else. */
|
|
10560
|
+
function frontmatterIdentity(text) {
|
|
10561
|
+
const span = frontmatterSpan(text);
|
|
10562
|
+
const data = span ? frontmatterData(span.body) : {};
|
|
10563
|
+
if (data === null) return {
|
|
10564
|
+
title: null,
|
|
10565
|
+
aliases: [],
|
|
10566
|
+
hasAliasesKey: false,
|
|
10567
|
+
hasBlock: true,
|
|
10568
|
+
readable: false
|
|
10569
|
+
};
|
|
10570
|
+
return {
|
|
10571
|
+
title: titleOf(data),
|
|
10572
|
+
aliases: aliasesOf({ data }),
|
|
10573
|
+
hasAliasesKey: "aliases" in data,
|
|
10574
|
+
hasBlock: span !== null,
|
|
10575
|
+
readable: true
|
|
10576
|
+
};
|
|
10577
|
+
}
|
|
9507
10578
|
function titleOf(data) {
|
|
9508
10579
|
const title = data.title;
|
|
9509
10580
|
return typeof title === "string" && title.trim() !== "" ? title : null;
|
|
@@ -9540,51 +10611,110 @@ function sameAliases(a, b) {
|
|
|
9540
10611
|
*/
|
|
9541
10612
|
function withFrontmatterIdentity(text, patch, options = {}) {
|
|
9542
10613
|
const span = frontmatterSpan(text);
|
|
9543
|
-
if (
|
|
9544
|
-
|
|
9545
|
-
|
|
9546
|
-
|
|
9547
|
-
const
|
|
9548
|
-
if (
|
|
9549
|
-
|
|
9550
|
-
|
|
9551
|
-
|
|
9552
|
-
|
|
9553
|
-
|
|
9554
|
-
|
|
9555
|
-
|
|
9556
|
-
|
|
9557
|
-
|
|
9558
|
-
|
|
9559
|
-
|
|
9560
|
-
|
|
10614
|
+
if (span) {
|
|
10615
|
+
const data = frontmatterData(span.body);
|
|
10616
|
+
if (data === null) return text;
|
|
10617
|
+
const wantTitle = patch.title === void 0 ? titleOf(data) : patch.title;
|
|
10618
|
+
const wantAliases = patch.aliases === void 0 ? aliasesOf({ data }) : patch.aliases;
|
|
10619
|
+
if (wantTitle === titleOf(data) && sameAliases(wantAliases, aliasesOf({ data }))) return text;
|
|
10620
|
+
}
|
|
10621
|
+
return editFrontmatter(text, (block) => {
|
|
10622
|
+
if (patch.title !== void 0) if (patch.title === null || patch.title.trim() === "") block.delete("title");
|
|
10623
|
+
else block.set("title", patch.title, "first");
|
|
10624
|
+
if (patch.aliases !== void 0) if (patch.aliases.length > 0) block.set("aliases", [...patch.aliases]);
|
|
10625
|
+
else block.delete("aliases");
|
|
10626
|
+
}, options);
|
|
10627
|
+
}
|
|
10628
|
+
/**
|
|
10629
|
+
* `after` - a writer's rewrite of `before` - with the document's aliases carried into the block
|
|
10630
|
+
* the rewrite added. A writer that gives a document its first block, as Publish does when it adds
|
|
10631
|
+
* `public:`, must not make the document claim fewer aliases than it has: a block with no
|
|
10632
|
+
* `aliases:` line claims none, and the next edit to it would clear them (ADR 0061). Only a Server
|
|
10633
|
+
* Backend has aliases without a block; on a Filesystem Backend they are read from the saved file,
|
|
10634
|
+
* so `aliases` is empty there provided the caller wrote pending edits before reading them
|
|
10635
|
+
* (`rewriteFrontmatter` flushes first). A block `before` already had is left as the writer left it.
|
|
10636
|
+
*/
|
|
10637
|
+
function withAliasesInAddedBlock(before, after, aliases) {
|
|
10638
|
+
if (aliases.length === 0 || frontmatterSpan(before) !== null || frontmatterSpan(after) === null) return after;
|
|
10639
|
+
return withFrontmatterIdentity(after, { aliases });
|
|
10640
|
+
}
|
|
10641
|
+
//#endregion
|
|
10642
|
+
//#region ../client/src/lib/document/properties.ts
|
|
10643
|
+
/**
|
|
10644
|
+
* Keys a Property Filter answers from the graph's names rather than the block: the registry on
|
|
10645
|
+
* a Server Backend holds them, and a page created in the app there has no block to read.
|
|
10646
|
+
*/
|
|
10647
|
+
var NAME_PROPERTY_KEYS = new Set(["title", "aliases"]);
|
|
10648
|
+
/** Rows per document at most. Frontmatter is metadata; a block past this is not a label set. */
|
|
10649
|
+
var MAX_ROWS = 256;
|
|
10650
|
+
/** Characters of one value kept. Longer text is prose, which Search's text group covers. */
|
|
10651
|
+
var MAX_VALUE_LENGTH = 1e3;
|
|
10652
|
+
/**
|
|
10653
|
+
* Whether a value counts as not set (ADR 0108): null, blank text, a list with no set item, or a
|
|
10654
|
+
* mapping with no set value. `false` and `0` are values.
|
|
10655
|
+
*/
|
|
10656
|
+
function isEmptyPropertyValue(value) {
|
|
10657
|
+
if (value === null || value === void 0) return true;
|
|
10658
|
+
if (typeof value === "string") return value.trim() === "";
|
|
10659
|
+
if (Array.isArray(value)) return value.every(isEmptyPropertyValue);
|
|
10660
|
+
if (isMapping(value)) return Object.values(value).every(isEmptyPropertyValue);
|
|
10661
|
+
return false;
|
|
10662
|
+
}
|
|
10663
|
+
/**
|
|
10664
|
+
* The rows for one document's parsed Frontmatter, in the block's order. Empty values are left
|
|
10665
|
+
* out, and so are `title` and `aliases` in any spelling: those filters read the graph's names.
|
|
10666
|
+
*/
|
|
10667
|
+
function propertiesOf(data) {
|
|
10668
|
+
const out = [];
|
|
9561
10669
|
for (const [key, value] of Object.entries(data)) {
|
|
9562
|
-
if (key
|
|
9563
|
-
|
|
9564
|
-
|
|
9565
|
-
|
|
9566
|
-
|
|
9567
|
-
|
|
9568
|
-
|
|
9569
|
-
|
|
9570
|
-
|
|
10670
|
+
if (NAME_PROPERTY_KEYS.has(key.toLowerCase())) continue;
|
|
10671
|
+
collect(key, value, out);
|
|
10672
|
+
if (out.length >= MAX_ROWS) break;
|
|
10673
|
+
}
|
|
10674
|
+
return out.slice(0, MAX_ROWS);
|
|
10675
|
+
}
|
|
10676
|
+
/** Append the rows for `value` under `path`; a mapping adds its presence row before its values. */
|
|
10677
|
+
function collect(path, value, out) {
|
|
10678
|
+
if (isEmptyPropertyValue(value)) return;
|
|
10679
|
+
if (Array.isArray(value)) {
|
|
10680
|
+
let presence = false;
|
|
10681
|
+
for (const item of value) if (isMapping(item)) {
|
|
10682
|
+
if (!presence && !isEmptyPropertyValue(item)) {
|
|
10683
|
+
out.push({
|
|
10684
|
+
key: path,
|
|
10685
|
+
value: null
|
|
10686
|
+
});
|
|
10687
|
+
presence = true;
|
|
10688
|
+
}
|
|
10689
|
+
for (const [key, child] of Object.entries(item)) collect(`${path}.${key}`, child, out);
|
|
10690
|
+
} else collect(path, item, out);
|
|
10691
|
+
return;
|
|
9571
10692
|
}
|
|
9572
|
-
if (
|
|
9573
|
-
|
|
9574
|
-
|
|
9575
|
-
|
|
9576
|
-
|
|
9577
|
-
|
|
9578
|
-
|
|
9579
|
-
if (parsed === null || parsed === void 0) return {};
|
|
9580
|
-
if (typeof parsed !== "object" || Array.isArray(parsed)) return null;
|
|
9581
|
-
return parsed;
|
|
9582
|
-
} catch {
|
|
9583
|
-
return null;
|
|
10693
|
+
if (isMapping(value)) {
|
|
10694
|
+
out.push({
|
|
10695
|
+
key: path,
|
|
10696
|
+
value: null
|
|
10697
|
+
});
|
|
10698
|
+
for (const [key, child] of Object.entries(value)) collect(`${path}.${key}`, child, out);
|
|
10699
|
+
return;
|
|
9584
10700
|
}
|
|
10701
|
+
out.push({
|
|
10702
|
+
key: path,
|
|
10703
|
+
value: textOf(value).slice(0, MAX_VALUE_LENGTH)
|
|
10704
|
+
});
|
|
10705
|
+
}
|
|
10706
|
+
function isMapping(value) {
|
|
10707
|
+
return typeof value === "object" && value !== null && !Array.isArray(value) && !(value instanceof Date);
|
|
9585
10708
|
}
|
|
9586
|
-
|
|
9587
|
-
|
|
10709
|
+
/** A scalar as the text a person would type to find it. */
|
|
10710
|
+
function textOf(value) {
|
|
10711
|
+
if (value instanceof Date) {
|
|
10712
|
+
const iso = value.toISOString();
|
|
10713
|
+
return iso.endsWith("T00:00:00.000Z") ? iso.slice(0, 10) : iso;
|
|
10714
|
+
}
|
|
10715
|
+
if (typeof value === "string") return value;
|
|
10716
|
+
if (typeof value === "number" || typeof value === "boolean" || typeof value === "bigint") return String(value);
|
|
10717
|
+
return JSON.stringify(value) ?? "";
|
|
9588
10718
|
}
|
|
9589
10719
|
//#endregion
|
|
9590
10720
|
//#region ../client/src/lib/document/calendar/month-grid-core.ts
|
|
@@ -10019,6 +11149,31 @@ async function refuseProtectedMerges(plan, isProtected) {
|
|
|
10019
11149
|
return plan;
|
|
10020
11150
|
}
|
|
10021
11151
|
/**
|
|
11152
|
+
* Refuse a plan whose steps would rewrite a [[Frontmatter]] block that does not parse. A
|
|
11153
|
+
* Filesystem Backend rebuilds each renamed document's block, and a merge survivor's, from its
|
|
11154
|
+
* parsed data, which is empty for such a block, so every key it holds would be lost from the file:
|
|
11155
|
+
* the only copy. `textOf` gives a document's current text (an open buffer, else its file), or
|
|
11156
|
+
* null for a name with no document behind it.
|
|
11157
|
+
*/
|
|
11158
|
+
async function refuseUnreadableBlocks(plan, textOf) {
|
|
11159
|
+
if (plan.refusal) return plan;
|
|
11160
|
+
for (const step of renameSteps(plan)) for (const concept of step.merges ? [step.from, step.into] : [step.from]) {
|
|
11161
|
+
const text = await textOf(concept);
|
|
11162
|
+
if (text === null || frontmatterIdentity(text).readable) continue;
|
|
11163
|
+
return {
|
|
11164
|
+
...plan,
|
|
11165
|
+
refusal: unreadableBlockRefusal(plan.direct.from, concept)
|
|
11166
|
+
};
|
|
11167
|
+
}
|
|
11168
|
+
return plan;
|
|
11169
|
+
}
|
|
11170
|
+
/** Why `renamed` cannot be renamed: the block of `unreadable` (itself, or a page the rename rewrites) does not parse. */
|
|
11171
|
+
function unreadableBlockRefusal(renamed, unreadable) {
|
|
11172
|
+
const fix = "Fix the block (a property written twice, or a line left half typed) and rename again.";
|
|
11173
|
+
if (conceptKey(renamed) === conceptKey(unreadable)) return `“${renamed}” cannot be renamed while its frontmatter is not valid YAML: renaming rewrites the block, and what it holds would be lost. ${fix}`;
|
|
11174
|
+
return `“${renamed}” cannot be renamed while the frontmatter of “${unreadable}”, which the rename rewrites, is not valid YAML: what it holds would be lost. ${fix}`;
|
|
11175
|
+
}
|
|
11176
|
+
/**
|
|
10022
11177
|
* A step merges when a DIFFERENT document already answers to the target name - by title or by
|
|
10023
11178
|
* alias - and the source has a document to join to it. Renaming onto yourself (a pure
|
|
10024
11179
|
* re-casing, or onto one of your own aliases) is neither. A source with no document landing
|
|
@@ -10140,23 +11295,6 @@ async function scanGraph(adapter, options = {}) {
|
|
|
10140
11295
|
//#endregion
|
|
10141
11296
|
//#region ../client/src/lib/storage/fs/filesystem-store.ts
|
|
10142
11297
|
/**
|
|
10143
|
-
* A {@link DocumentStore} over a real directory (via a {@link DirectoryAdapter}):
|
|
10144
|
-
* the Filesystem Backend. The files *are* the state — this holds only a live
|
|
10145
|
-
* buffer per open document plus a derived registry, and reconciles external
|
|
10146
|
-
* changes (git checkout, Syncthing) the way DESIGN.md → The git workflow requires.
|
|
10147
|
-
*
|
|
10148
|
-
* Assembled entirely from the pure pieces (scan, identity, reconcileDecision,
|
|
10149
|
-
* debounce) so it is exercised in Node over createMemoryDirectoryAdapter before
|
|
10150
|
-
* any browser code exists.
|
|
10151
|
-
*
|
|
10152
|
-
* Async-seam note: the seam's
|
|
10153
|
-
* `getText()` is synchronous but disk reads are async, so `open()` returns a
|
|
10154
|
-
* handle whose buffer is empty on first open and is hydrated by an internal
|
|
10155
|
-
* awaited read that then notifies subscribers the *external* way — which is safe
|
|
10156
|
-
* because DocumentView registers its `subscribe` listener in the same synchronous
|
|
10157
|
-
* onMount tick as its `getText()` seed, before the read resolves.
|
|
10158
|
-
*/
|
|
10159
|
-
/**
|
|
10160
11298
|
* The snapshot for one document's full text: identity from the registry, aliases and include
|
|
10161
11299
|
* facts from the frontmatter, the body for the index. Both backends build theirs this way, so
|
|
10162
11300
|
* the index sees one shape whatever holds the document.
|
|
@@ -10169,12 +11307,14 @@ function indexSnapshotOf(identity, text) {
|
|
|
10169
11307
|
text,
|
|
10170
11308
|
aliases: []
|
|
10171
11309
|
}) : [];
|
|
11310
|
+
const properties = propertiesOf(fm.data);
|
|
10172
11311
|
return {
|
|
10173
11312
|
concept: identity.concept,
|
|
10174
11313
|
kind: identity.kind,
|
|
10175
11314
|
aliases: aliasesOf(fm),
|
|
10176
11315
|
text: fm.body,
|
|
10177
|
-
...includes.length > 0 ? { includes } : {}
|
|
11316
|
+
...includes.length > 0 ? { includes } : {},
|
|
11317
|
+
...properties.length > 0 ? { properties } : {}
|
|
10178
11318
|
};
|
|
10179
11319
|
}
|
|
10180
11320
|
function applyTextChange(text, change) {
|
|
@@ -10343,17 +11483,20 @@ function createFilesystemDocumentStore(adapter, options = {}) {
|
|
|
10343
11483
|
await settled(openDoc);
|
|
10344
11484
|
}
|
|
10345
11485
|
const { text } = await adapter.read(entry.subdir, entry.fileName);
|
|
11486
|
+
if (!frontmatterIdentity(text).readable) throw new Error(unreadableBlockRefusal(step.from, entry.concept));
|
|
10346
11487
|
const fm = parseFrontmatter(text);
|
|
10347
11488
|
let aliases = aliasesOf(fm);
|
|
10348
11489
|
let body = fm.body;
|
|
10349
|
-
let
|
|
11490
|
+
let blockSource = text;
|
|
10350
11491
|
if (strategy === "alias") {
|
|
10351
11492
|
if (!aliases.some((a) => conceptKey(a) === conceptKey(step.from))) aliases = [...aliases, step.from];
|
|
10352
11493
|
}
|
|
10353
11494
|
const targetEntry = step.merges ? registry.get(conceptKey(step.into)) : void 0;
|
|
10354
11495
|
const survivor = targetEntry && targetEntry.key !== entry.key ? targetEntry : void 0;
|
|
10355
11496
|
if (survivor) {
|
|
10356
|
-
const
|
|
11497
|
+
const existing = await adapter.read(survivor.subdir, survivor.fileName);
|
|
11498
|
+
if (!frontmatterIdentity(existing.text).readable) throw new Error(unreadableBlockRefusal(step.from, survivor.concept));
|
|
11499
|
+
const existingFm = parseFrontmatter(existing.text);
|
|
10357
11500
|
const merged = mergeDocuments({
|
|
10358
11501
|
body: existingFm.body,
|
|
10359
11502
|
aliases: aliasesOf(existingFm)
|
|
@@ -10363,15 +11506,17 @@ function createFilesystemDocumentStore(adapter, options = {}) {
|
|
|
10363
11506
|
});
|
|
10364
11507
|
body = merged.body;
|
|
10365
11508
|
aliases = merged.aliases;
|
|
10366
|
-
|
|
11509
|
+
blockSource = existing.text;
|
|
10367
11510
|
}
|
|
10368
11511
|
aliases = normaliseAliases(aliases, step.into);
|
|
10369
|
-
|
|
10370
|
-
|
|
10371
|
-
|
|
11512
|
+
const blockSpan = frontmatterSpan(blockSource);
|
|
11513
|
+
const content = withFrontmatterIdentity((blockSpan ? blockSource.slice(0, blockSpan.end) : "") + body, {
|
|
11514
|
+
title: step.into,
|
|
11515
|
+
aliases
|
|
11516
|
+
}, { addBlock: true });
|
|
10372
11517
|
const subdir = survivor?.subdir ?? entry.subdir;
|
|
10373
11518
|
const fileName = survivor?.fileName ?? await allocateFileName(subdir, step.into, entry);
|
|
10374
|
-
const written = await adapter.write(subdir, fileName,
|
|
11519
|
+
const written = await adapter.write(subdir, fileName, content);
|
|
10375
11520
|
if (!(subdir === entry.subdir && fileName === entry.fileName)) await adapter.remove(entry.subdir, entry.fileName);
|
|
10376
11521
|
registry.delete(entry.key);
|
|
10377
11522
|
if (survivor) registry.delete(survivor.key);
|
|
@@ -10462,6 +11607,29 @@ function createFilesystemDocumentStore(adapter, options = {}) {
|
|
|
10462
11607
|
if (doc.dirty) runSave(doc);
|
|
10463
11608
|
await settled(doc);
|
|
10464
11609
|
}
|
|
11610
|
+
/**
|
|
11611
|
+
* The registry entry `target` names: a document by its own name, else by one of its aliases
|
|
11612
|
+
* (ADR 0061), as a bookmark or a Recents entry can carry a renamed page's old name. A page's own
|
|
11613
|
+
* name outranks another page's alias of the same name, as the index resolves it.
|
|
11614
|
+
*/
|
|
11615
|
+
function entryNamed(target) {
|
|
11616
|
+
const key = conceptKey(target);
|
|
11617
|
+
const own = registry.get(key);
|
|
11618
|
+
if (own) return own;
|
|
11619
|
+
for (const entry of registry.values()) if (entry.aliases.some((alias) => conceptKey(alias) === key)) return entry;
|
|
11620
|
+
}
|
|
11621
|
+
/** The open document `target` names, opening it if it is not; throws when nothing has that name. */
|
|
11622
|
+
function openDocNamed(target) {
|
|
11623
|
+
const existing = open.get(conceptKey(target));
|
|
11624
|
+
if (existing) return existing;
|
|
11625
|
+
const entry = entryNamed(target);
|
|
11626
|
+
if (!entry) throw new DocumentNotFoundError(target);
|
|
11627
|
+
const opened = open.get(entry.key);
|
|
11628
|
+
if (opened) return opened;
|
|
11629
|
+
const doc = makeOpenDoc(entry.key === conceptKey(target) ? target : entry.concept, entry);
|
|
11630
|
+
open.set(entry.key, doc);
|
|
11631
|
+
return doc;
|
|
11632
|
+
}
|
|
10465
11633
|
function makeOpenDoc(target, entry) {
|
|
10466
11634
|
const doc = {
|
|
10467
11635
|
key: entry.key,
|
|
@@ -10565,17 +11733,10 @@ function createFilesystemDocumentStore(adapter, options = {}) {
|
|
|
10565
11733
|
}
|
|
10566
11734
|
return {
|
|
10567
11735
|
open(target) {
|
|
10568
|
-
|
|
10569
|
-
const existing = open.get(key);
|
|
10570
|
-
if (existing) return existing.handle;
|
|
10571
|
-
const entry = registry.get(key);
|
|
10572
|
-
if (!entry) throw new DocumentNotFoundError(target);
|
|
10573
|
-
const doc = makeOpenDoc(target, entry);
|
|
10574
|
-
open.set(key, doc);
|
|
10575
|
-
return doc.handle;
|
|
11736
|
+
return openDocNamed(target).handle;
|
|
10576
11737
|
},
|
|
10577
11738
|
async whenReady(target) {
|
|
10578
|
-
await (
|
|
11739
|
+
await openDocNamed(target).ready;
|
|
10579
11740
|
},
|
|
10580
11741
|
async scan() {
|
|
10581
11742
|
await adapter.ensureSkeleton();
|
|
@@ -10718,13 +11879,13 @@ function createFilesystemDocumentStore(adapter, options = {}) {
|
|
|
10718
11879
|
if (registry.get(key)) throw new Error(`A document for "${concept}" already exists.`);
|
|
10719
11880
|
await adapter.ensureSkeleton();
|
|
10720
11881
|
const fileName = await allocateFileName("pages", concept);
|
|
10721
|
-
const content =
|
|
11882
|
+
const content = renderFrontmatter({ title: concept }) + body;
|
|
10722
11883
|
await adapter.write("pages", fileName, content);
|
|
10723
11884
|
await refreshRegistry();
|
|
10724
11885
|
return concept;
|
|
10725
11886
|
},
|
|
10726
11887
|
async planRename(from, to, referencingDocuments) {
|
|
10727
|
-
return refuseProtectedMerges(planRename$1({
|
|
11888
|
+
return refuseUnreadableBlocks(await refuseProtectedMerges(planRename$1({
|
|
10728
11889
|
from,
|
|
10729
11890
|
to,
|
|
10730
11891
|
kind: registry.get(conceptKey(from))?.kind ?? null,
|
|
@@ -10738,6 +11899,12 @@ function createFilesystemDocumentStore(adapter, options = {}) {
|
|
|
10738
11899
|
const other = registry.get(conceptKey(concept));
|
|
10739
11900
|
if (!other) return false;
|
|
10740
11901
|
return documentProtection(open.get(other.key)?.buffer ?? (await adapter.read(other.subdir, other.fileName)).text).kind === "document";
|
|
11902
|
+
}), async (concept) => {
|
|
11903
|
+
const other = registry.get(conceptKey(concept));
|
|
11904
|
+
if (!other) return null;
|
|
11905
|
+
const openDoc = open.get(other.key);
|
|
11906
|
+
if (openDoc) await settled(openDoc);
|
|
11907
|
+
return (await adapter.read(other.subdir, other.fileName)).text;
|
|
10741
11908
|
});
|
|
10742
11909
|
},
|
|
10743
11910
|
async renamePage(from, to, options) {
|
|
@@ -10800,6 +11967,11 @@ function createFilesystemDocumentStore(adapter, options = {}) {
|
|
|
10800
11967
|
if (!doc) return;
|
|
10801
11968
|
await saveNow(doc);
|
|
10802
11969
|
},
|
|
11970
|
+
async flushAll() {
|
|
11971
|
+
await Promise.all([...open.values()].map(saveNow));
|
|
11972
|
+
const unwritten = [...open.values()].filter((doc) => doc.dirty).map((doc) => doc.target);
|
|
11973
|
+
if (unwritten.length > 0) throw new Error(`Edits to ${unwritten.join(", ")} could not be written to the folder yet, so it does not hold them. Try again once they are saved.`);
|
|
11974
|
+
},
|
|
10803
11975
|
async dispose() {
|
|
10804
11976
|
await Promise.all([...open.values()].map(saveNow));
|
|
10805
11977
|
open.clear();
|
|
@@ -11000,10 +12172,6 @@ async function reachedWithin(promise, timeoutMs) {
|
|
|
11000
12172
|
if (timer) clearTimeout(timer);
|
|
11001
12173
|
}
|
|
11002
12174
|
}
|
|
11003
|
-
/** Health that says the bytes are not this document's, so neither text nor emptiness is real. */
|
|
11004
|
-
function contentBlocked(health) {
|
|
11005
|
-
return health === "key-unavailable" || health === "ciphertext-corrupt";
|
|
11006
|
-
}
|
|
11007
12175
|
function createServerDocumentStore(graph, options) {
|
|
11008
12176
|
const registry = graph.registry();
|
|
11009
12177
|
/** Engines created between yields while snapshotting — small enough to keep frames free. */
|
|
@@ -11014,6 +12182,9 @@ function createServerDocumentStore(graph, options) {
|
|
|
11014
12182
|
const changeListeners = /* @__PURE__ */ new Set();
|
|
11015
12183
|
const docsChangedListeners = /* @__PURE__ */ new Set();
|
|
11016
12184
|
const removedListeners = /* @__PURE__ */ new Set();
|
|
12185
|
+
const renamedListeners = /* @__PURE__ */ new Set();
|
|
12186
|
+
/** docId to the name it had when first seen renamed from elsewhere, and its latest name. */
|
|
12187
|
+
const renamesFromElsewhere = /* @__PURE__ */ new Map();
|
|
11017
12188
|
const resurrectionClaims = /* @__PURE__ */ new Map();
|
|
11018
12189
|
let disposed = false;
|
|
11019
12190
|
/** Whether the encrypted registry has been read to its terminal relay page this session. */
|
|
@@ -11033,9 +12204,14 @@ function createServerDocumentStore(graph, options) {
|
|
|
11033
12204
|
*/
|
|
11034
12205
|
async function materialise(docIds, timeoutMs) {
|
|
11035
12206
|
const unique = [...new Set(docIds)];
|
|
11036
|
-
const
|
|
12207
|
+
const held = [];
|
|
12208
|
+
const release = () => graph.retireDocs(held);
|
|
11037
12209
|
try {
|
|
11038
|
-
for (let start = 0; start < unique.length; start += ENGINE_BATCH)
|
|
12210
|
+
for (let start = 0; start < unique.length; start += ENGINE_BATCH) {
|
|
12211
|
+
const batch = unique.slice(start, start + ENGINE_BATCH);
|
|
12212
|
+
held.push(...batch);
|
|
12213
|
+
await graph.readyDocs(batch);
|
|
12214
|
+
}
|
|
11039
12215
|
const connected = graph.isConnected();
|
|
11040
12216
|
const behind = new Set(connected ? await graph.docsNeedingCatchup(unique) : []);
|
|
11041
12217
|
const unconfirmed = [];
|
|
@@ -11092,11 +12268,18 @@ function createServerDocumentStore(graph, options) {
|
|
|
11092
12268
|
});
|
|
11093
12269
|
return out;
|
|
11094
12270
|
}
|
|
12271
|
+
/**
|
|
12272
|
+
* Every name a document answers to, to its docId. A page's own name outranks another page's
|
|
12273
|
+
* alias of the same name, whatever order the registry lists them in, as the index resolves it.
|
|
12274
|
+
*/
|
|
11095
12275
|
function currentDocIds() {
|
|
11096
12276
|
const out = /* @__PURE__ */ new Map();
|
|
12277
|
+
registry.forEach((entry, docId) => out.set(conceptKey(conceptOf(entry)), docId));
|
|
11097
12278
|
registry.forEach((entry, docId) => {
|
|
11098
|
-
|
|
11099
|
-
|
|
12279
|
+
for (const alias of entry.aliases ?? []) {
|
|
12280
|
+
const key = conceptKey(alias);
|
|
12281
|
+
if (!out.has(key)) out.set(key, docId);
|
|
12282
|
+
}
|
|
11100
12283
|
});
|
|
11101
12284
|
return out;
|
|
11102
12285
|
}
|
|
@@ -11105,25 +12288,43 @@ function createServerDocumentStore(graph, options) {
|
|
|
11105
12288
|
const nextRegistry = currentRegistryIdentities();
|
|
11106
12289
|
const changedDocIds = [...new Set([...knownRegistry.keys(), ...nextRegistry.keys()])].filter((docId) => !sameIdentity(knownRegistry.get(docId), nextRegistry.get(docId)));
|
|
11107
12290
|
if (changedDocIds.length === 0) return;
|
|
12291
|
+
const restored = event.transaction.origin === CACHE_SEED;
|
|
12292
|
+
const fromElsewhere = !restored && !event.transaction.local;
|
|
11108
12293
|
const additions = [];
|
|
12294
|
+
const renames = [];
|
|
11109
12295
|
let additionsOnly = true;
|
|
11110
12296
|
for (const docId of changedDocIds) {
|
|
11111
12297
|
const before = knownRegistry.get(docId);
|
|
11112
12298
|
const after = nextRegistry.get(docId);
|
|
12299
|
+
if (fromElsewhere && before && after && before.concept !== after.concept) renames.push({
|
|
12300
|
+
docId,
|
|
12301
|
+
from: before.concept,
|
|
12302
|
+
to: after.concept
|
|
12303
|
+
});
|
|
11113
12304
|
if (before || !after) {
|
|
11114
12305
|
additionsOnly = false;
|
|
11115
12306
|
continue;
|
|
11116
12307
|
}
|
|
11117
12308
|
additions.push(after.concept);
|
|
11118
12309
|
}
|
|
12310
|
+
const renamedAway = new Set(renames.map((rename) => conceptKey(rename.from)));
|
|
11119
12311
|
const next = currentConcepts();
|
|
11120
|
-
for (const [key, concept] of known) if (!next.has(key)) removedListeners.forEach((l) => l(concept));
|
|
12312
|
+
for (const [key, concept] of known) if (!next.has(key) && !renamedAway.has(key)) removedListeners.forEach((l) => l(concept));
|
|
11121
12313
|
known = next;
|
|
11122
12314
|
docIdsByConcept = currentDocIds();
|
|
11123
12315
|
knownRegistry = nextRegistry;
|
|
11124
12316
|
docsChangedListeners.forEach((l) => l());
|
|
11125
|
-
if (additionsOnly) for (const concept of additions) changeListeners.forEach((listener) => listener({ concept }));
|
|
12317
|
+
if (!restored) if (additionsOnly) for (const concept of additions) changeListeners.forEach((listener) => listener({ concept }));
|
|
11126
12318
|
else changeListeners.forEach((listener) => listener());
|
|
12319
|
+
for (const { docId, from, to } of renames) {
|
|
12320
|
+
const first = renamesFromElsewhere.get(docId)?.from ?? from;
|
|
12321
|
+
if (first === to) renamesFromElsewhere.delete(docId);
|
|
12322
|
+
else renamesFromElsewhere.set(docId, {
|
|
12323
|
+
from: first,
|
|
12324
|
+
to
|
|
12325
|
+
});
|
|
12326
|
+
renamedListeners.forEach((listener) => listener(from, to));
|
|
12327
|
+
}
|
|
11127
12328
|
for (const [docId, change] of event.changes.keys) {
|
|
11128
12329
|
if (change.action !== "delete") continue;
|
|
11129
12330
|
const claimed = resurrectionClaims.get(docId);
|
|
@@ -11171,12 +12372,14 @@ function createServerDocumentStore(graph, options) {
|
|
|
11171
12372
|
text,
|
|
11172
12373
|
aliases: []
|
|
11173
12374
|
}) : [];
|
|
12375
|
+
const properties = propertiesOf(parseFrontmatter(text).data);
|
|
11174
12376
|
return {
|
|
11175
12377
|
concept,
|
|
11176
12378
|
kind: entry.kind,
|
|
11177
12379
|
aliases: entry.aliases ?? [],
|
|
11178
12380
|
text: bodyForIndex(text),
|
|
11179
|
-
...includes.length > 0 ? { includes } : {}
|
|
12381
|
+
...includes.length > 0 ? { includes } : {},
|
|
12382
|
+
...properties.length > 0 ? { properties } : {}
|
|
11180
12383
|
};
|
|
11181
12384
|
}
|
|
11182
12385
|
/**
|
|
@@ -11401,6 +12604,18 @@ function createServerDocumentStore(graph, options) {
|
|
|
11401
12604
|
}
|
|
11402
12605
|
return out;
|
|
11403
12606
|
},
|
|
12607
|
+
async docsBehind(docIds, timeoutMs = COLD_CONTENT_TIMEOUT_MS) {
|
|
12608
|
+
if (disposed || !graph.isConnected()) return null;
|
|
12609
|
+
if (docIds.length === 0) return [];
|
|
12610
|
+
let timer;
|
|
12611
|
+
try {
|
|
12612
|
+
return await Promise.race([graph.docsNeedingCatchup(docIds).catch(() => null), new Promise((resolve) => {
|
|
12613
|
+
timer = setTimeout(() => resolve(null), timeoutMs);
|
|
12614
|
+
})]);
|
|
12615
|
+
} finally {
|
|
12616
|
+
if (timer) clearTimeout(timer);
|
|
12617
|
+
}
|
|
12618
|
+
},
|
|
11404
12619
|
async confirmRegistry(timeoutMs = 5e3) {
|
|
11405
12620
|
if (registryConfirmed) return true;
|
|
11406
12621
|
if (!graph.isConnected()) return false;
|
|
@@ -11415,6 +12630,13 @@ function createServerDocumentStore(graph, options) {
|
|
|
11415
12630
|
removedListeners.add(listener);
|
|
11416
12631
|
return () => removedListeners.delete(listener);
|
|
11417
12632
|
},
|
|
12633
|
+
onDocumentRenamed(listener) {
|
|
12634
|
+
renamedListeners.add(listener);
|
|
12635
|
+
return () => renamedListeners.delete(listener);
|
|
12636
|
+
},
|
|
12637
|
+
renamesObserved() {
|
|
12638
|
+
return [...renamesFromElsewhere.values()].map((rename) => ({ ...rename }));
|
|
12639
|
+
},
|
|
11418
12640
|
onChange(listener) {
|
|
11419
12641
|
changeListeners.add(listener);
|
|
11420
12642
|
return () => changeListeners.delete(listener);
|
|
@@ -11528,11 +12750,16 @@ function createServerDocumentStore(graph, options) {
|
|
|
11528
12750
|
acknowledge: checkpoint.acknowledge
|
|
11529
12751
|
};
|
|
11530
12752
|
},
|
|
11531
|
-
snapshotDocument(concept) {
|
|
12753
|
+
async snapshotDocument(concept) {
|
|
11532
12754
|
const docId = docIdFor(concept);
|
|
11533
|
-
|
|
11534
|
-
|
|
11535
|
-
|
|
12755
|
+
if (!docId || !registry.has(docId)) return null;
|
|
12756
|
+
try {
|
|
12757
|
+
await graph.readyDocs([docId]);
|
|
12758
|
+
const entry = registry.get(docId);
|
|
12759
|
+
return entry ? indexSnapshotFor(entry, graph.docSync(docId).doc.getText("content").toString()) : null;
|
|
12760
|
+
} finally {
|
|
12761
|
+
graph.retireDocs([docId]);
|
|
12762
|
+
}
|
|
11536
12763
|
},
|
|
11537
12764
|
async compactDocument(target) {
|
|
11538
12765
|
const docId = docIdFor(target);
|
|
@@ -11799,7 +13026,7 @@ async function withRetry(task, options = {}) {
|
|
|
11799
13026
|
} catch (error) {
|
|
11800
13027
|
if (options.signal?.aborted) throw options.signal.reason;
|
|
11801
13028
|
if (attempt >= attempts || !shouldRetry(error)) throw error;
|
|
11802
|
-
const full = base * 2 ** (attempt - 1);
|
|
13029
|
+
const full = Math.min(base * 2 ** (attempt - 1), options.maxDelayMs ?? Number.POSITIVE_INFINITY);
|
|
11803
13030
|
const wait = Math.round(full / 2 + random() * (full / 2));
|
|
11804
13031
|
options.onRetry?.(error, attempt, wait);
|
|
11805
13032
|
await sleep(wait, options.signal);
|
|
@@ -11889,7 +13116,11 @@ var HttpFailure = class extends Error {
|
|
|
11889
13116
|
*/
|
|
11890
13117
|
function isTransient(error) {
|
|
11891
13118
|
if (!(error instanceof HttpFailure)) return true;
|
|
11892
|
-
return error.status
|
|
13119
|
+
return isTransientStatus(error.status);
|
|
13120
|
+
}
|
|
13121
|
+
/** "Not your fault, try again": a 401 only once a fresh token has been presented. */
|
|
13122
|
+
function isTransientStatus(status) {
|
|
13123
|
+
return status === 401 || status === 408 || status === 429 || status >= 500;
|
|
11893
13124
|
}
|
|
11894
13125
|
function kebabStem(name) {
|
|
11895
13126
|
return name.replace(/\.[^.]+$/, "").toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "") || "asset";
|
|
@@ -11960,14 +13191,33 @@ function createServerAssetStore(deps) {
|
|
|
11960
13191
|
}, retryOptions);
|
|
11961
13192
|
}
|
|
11962
13193
|
/**
|
|
11963
|
-
*
|
|
11964
|
-
*
|
|
13194
|
+
* A read request whose connection failure is a file that cannot be fetched just now, not a
|
|
13195
|
+
* missing one. A cancellation stays a cancellation.
|
|
11965
13196
|
*/
|
|
11966
|
-
async function
|
|
13197
|
+
async function readRequest(url, init) {
|
|
13198
|
+
try {
|
|
13199
|
+
return await f(url, init);
|
|
13200
|
+
} catch (error) {
|
|
13201
|
+
if (error instanceof DOMException && error.name === "AbortError") throw error;
|
|
13202
|
+
throw new AssetUnavailableError("The file could not be fetched: the connection failed.", { cause: error });
|
|
13203
|
+
}
|
|
13204
|
+
}
|
|
13205
|
+
/**
|
|
13206
|
+
* Fetch, decrypt and reassemble one asset; both read paths go through here. `null` when the
|
|
13207
|
+
* server has no such asset or refuses it. A failure that says nothing about the asset rejects
|
|
13208
|
+
* with `AssetUnavailableError`: the connection, or a status that means "try again" (a 401
|
|
13209
|
+
* after one fresh token). A file that will not decrypt or parse rejects with that error.
|
|
13210
|
+
*/
|
|
13211
|
+
async function fetchAssetBytes(ref) {
|
|
11967
13212
|
const assetId = assetIdFromRef(ref);
|
|
11968
13213
|
if (!assetId) return null;
|
|
11969
|
-
const
|
|
11970
|
-
|
|
13214
|
+
const url = `${base}/api/v1/sync/assets/${deps.graphId}/${assetId}`;
|
|
13215
|
+
let res = await readRequest(url, { headers: { "x-sync-token": await deps.syncToken() } });
|
|
13216
|
+
if (res.status === 401) res = await readRequest(url, { headers: { "x-sync-token": await deps.syncToken({ force: true }) } });
|
|
13217
|
+
if (!res.ok) {
|
|
13218
|
+
if (isTransientStatus(res.status)) throw new AssetUnavailableError(`The sync server could not hand over the file just now (HTTP ${res.status}).`, { status: res.status });
|
|
13219
|
+
return null;
|
|
13220
|
+
}
|
|
11971
13221
|
const body = await res.json();
|
|
11972
13222
|
const metaPlain = await openSymmetric({
|
|
11973
13223
|
keyForEpoch: (id) => keyForEpoch(deps.keyring, id),
|
|
@@ -11978,8 +13228,11 @@ function createServerAssetStore(deps) {
|
|
|
11978
13228
|
const perAssetKey = fromBase64Url(metadata.perAssetKey);
|
|
11979
13229
|
const parts = [];
|
|
11980
13230
|
for (let n = 0; n < body.downloadUrls.length; n++) {
|
|
11981
|
-
const chunkRes = await
|
|
11982
|
-
if (!chunkRes.ok)
|
|
13231
|
+
const chunkRes = await readRequest(body.downloadUrls[n]);
|
|
13232
|
+
if (!chunkRes.ok) {
|
|
13233
|
+
if (chunkRes.status !== 401 && isTransientStatus(chunkRes.status)) throw new AssetUnavailableError(`Storage could not hand over the file just now (HTTP ${chunkRes.status}).`, { status: chunkRes.status });
|
|
13234
|
+
return null;
|
|
13235
|
+
}
|
|
11983
13236
|
const { plaintext } = await openSymmetric({
|
|
11984
13237
|
keyForEpoch: () => perAssetKey,
|
|
11985
13238
|
envelope: new Uint8Array(await chunkRes.arrayBuffer()),
|
|
@@ -12069,7 +13322,7 @@ function createServerAssetStore(deps) {
|
|
|
12069
13322
|
body: chunks[n]
|
|
12070
13323
|
});
|
|
12071
13324
|
} catch {
|
|
12072
|
-
throw new Error("Uploading to the storage bucket was blocked. The bucket's CORS policy must allow this app's origin - see \"Asset storage\" in the Synced Graphs doc.");
|
|
13325
|
+
throw new Error(typeof navigator !== "undefined" && navigator.onLine === false ? "The connection dropped while uploading to the storage bucket." : "Uploading to the storage bucket was blocked. The bucket's CORS policy must allow this app's origin - see \"Asset storage\" in the Synced Graphs doc.");
|
|
12073
13326
|
}
|
|
12074
13327
|
if (!r.ok) throw new HttpFailure(r.status, `chunk ${n} upload failed: ${r.status}`);
|
|
12075
13328
|
onBytes?.(Math.min(CHUNK_SIZE, bytes.length - n * CHUNK_SIZE));
|
|
@@ -12085,9 +13338,9 @@ function createServerAssetStore(deps) {
|
|
|
12085
13338
|
});
|
|
12086
13339
|
return saved(assetId, false);
|
|
12087
13340
|
},
|
|
12088
|
-
readBytes:
|
|
13341
|
+
readBytes: (ref) => fetchAssetBytes(ref).catch((error) => error instanceof AssetUnavailableError && error.status !== void 0 ? null : Promise.reject(error)),
|
|
12089
13342
|
async resolve(ref) {
|
|
12090
|
-
const asset = await
|
|
13343
|
+
const asset = await fetchAssetBytes(ref);
|
|
12091
13344
|
if (!asset) return null;
|
|
12092
13345
|
const url = URL.createObjectURL(new Blob([asset.bytes], { type: asset.type }));
|
|
12093
13346
|
objectUrls.push(url);
|
|
@@ -12129,6 +13382,20 @@ async function listServerAssets(deps) {
|
|
|
12129
13382
|
return assets;
|
|
12130
13383
|
}
|
|
12131
13384
|
//#endregion
|
|
13385
|
+
//#region ../client/src/lib/sync/write-refusal.ts
|
|
13386
|
+
var AGENT_REASONS = {
|
|
13387
|
+
entitlement_inactive: "the graph owner's plan does not allow changes at the moment (a lapsed, unpaid or unconfirmed plan)",
|
|
13388
|
+
owned_storage_limit: "the graph owner's storage allowance is used up"
|
|
13389
|
+
};
|
|
13390
|
+
/**
|
|
13391
|
+
* The Headless Client's refusal, for an agent: the edit was refused, not lost in transit, and it
|
|
13392
|
+
* needs no retry from the agent. The connection-loss wording ("will be delivered when the
|
|
13393
|
+
* connection recovers") is untrue of a refusal.
|
|
13394
|
+
*/
|
|
13395
|
+
function writeRefusalForAgent(refusal, outstanding) {
|
|
13396
|
+
return `The Sync Server refused the edit: ${(refusal.quotaCode && AGENT_REASONS[refusal.quotaCode]) ?? "a plan limit was reached"}. ${outstanding === 1 ? "1 operation is held" : `${outstanding} operations are held`} here and sent automatically once the server accepts changes again.`;
|
|
13397
|
+
}
|
|
13398
|
+
//#endregion
|
|
12132
13399
|
//#region src/headless-graph.ts
|
|
12133
13400
|
/**
|
|
12134
13401
|
* How often the cache and index are exported while a build is running. A build reports
|
|
@@ -12247,6 +13514,7 @@ function assembleHeadlessGraph(parts) {
|
|
|
12247
13514
|
publishing: parts.publishing,
|
|
12248
13515
|
themes: parts.themes,
|
|
12249
13516
|
settle: () => parts.settle(schedulePersist),
|
|
13517
|
+
accessLoss: () => parts.accessLoss?.() ?? null,
|
|
12250
13518
|
persist,
|
|
12251
13519
|
semantic,
|
|
12252
13520
|
semanticOpened: () => semanticOpening ? semanticOpening.catch(() => void 0) : Promise.resolve(void 0),
|
|
@@ -12264,8 +13532,9 @@ function assembleHeadlessGraph(parts) {
|
|
|
12264
13532
|
}
|
|
12265
13533
|
/** Open a synced graph, scan its registry and build the index; resolves once tools can answer. */
|
|
12266
13534
|
async function openHeadlessGraph(deps) {
|
|
12267
|
-
const
|
|
12268
|
-
|
|
13535
|
+
const cacheName = deps.localCacheName ?? deps.graphId;
|
|
13536
|
+
const cache = await openGraphCache(cacheName);
|
|
13537
|
+
if (deps.persistDir) await loadLocalCache(deps.persistDir, cacheName);
|
|
12269
13538
|
const indexHost = deps.persistDir ? nodeIndexHost(deps.persistDir) : void 0;
|
|
12270
13539
|
const sync = createGraphSync({
|
|
12271
13540
|
graphId: deps.graphId,
|
|
@@ -12282,6 +13551,7 @@ async function openHeadlessGraph(deps) {
|
|
|
12282
13551
|
...PRESENCE_PALETTE[0]
|
|
12283
13552
|
},
|
|
12284
13553
|
onError: deps.onError,
|
|
13554
|
+
onAccessLost: deps.onAccessLost,
|
|
12285
13555
|
publishName: deps.publishName
|
|
12286
13556
|
});
|
|
12287
13557
|
const store = createServerDocumentStore(sync, { readyTimeoutMs: deps.readyTimeoutMs });
|
|
@@ -12364,6 +13634,7 @@ async function openHeadlessGraph(deps) {
|
|
|
12364
13634
|
return assembleHeadlessGraph({
|
|
12365
13635
|
graphId: deps.graphId,
|
|
12366
13636
|
name: sync.getMeta().name ?? deps.graphId,
|
|
13637
|
+
accessLoss: () => sync.accessLoss(),
|
|
12367
13638
|
store: documents,
|
|
12368
13639
|
index,
|
|
12369
13640
|
assets,
|
|
@@ -12392,13 +13663,19 @@ async function openHeadlessGraph(deps) {
|
|
|
12392
13663
|
embeddingModel: deps.embeddingModel,
|
|
12393
13664
|
onSemanticProgress: deps.onSemanticProgress,
|
|
12394
13665
|
onError: deps.onError,
|
|
12395
|
-
persistBackend: deps.persistDir ? () => persistLocalCache(deps.persistDir,
|
|
13666
|
+
persistBackend: deps.persistDir ? () => persistLocalCache(deps.persistDir, cacheName) : void 0,
|
|
12396
13667
|
onChange: (schedule) => sync.onDocUpdate(schedule),
|
|
12397
13668
|
async settle(schedulePersist) {
|
|
12398
13669
|
await sync.flushAll();
|
|
12399
13670
|
const result = await sync.awaitAcked({ stallMs: 1e4 });
|
|
12400
13671
|
schedulePersist();
|
|
12401
13672
|
if (result.settled) return { settled: true };
|
|
13673
|
+
if (result.refused) return {
|
|
13674
|
+
settled: false,
|
|
13675
|
+
outstanding: result.outstanding,
|
|
13676
|
+
code: "write_refused",
|
|
13677
|
+
message: writeRefusalForAgent(result.refused, result.outstanding)
|
|
13678
|
+
};
|
|
12402
13679
|
return {
|
|
12403
13680
|
settled: false,
|
|
12404
13681
|
outstanding: result.outstanding,
|
|
@@ -12676,6 +13953,43 @@ async function openHeadlessFolder(deps) {
|
|
|
12676
13953
|
}
|
|
12677
13954
|
}
|
|
12678
13955
|
//#endregion
|
|
13956
|
+
//#region ../client/src/lib/sync/recovery-unlock.ts
|
|
13957
|
+
/**
|
|
13958
|
+
* Unlock with a Recovery Code, checked against the account's vault.
|
|
13959
|
+
*
|
|
13960
|
+
* A Recovery Code derives the vault's wrap key, and any well-formed code derives *a* key. Caching
|
|
13961
|
+
* whatever came out would report a mistyped or retired code as success, and every graph would
|
|
13962
|
+
* then fail to open with a message that blamed the keys on the device, next to the control that
|
|
13963
|
+
* resets them. The browser and the Headless Client both use this one check: fetch the vault,
|
|
13964
|
+
* open it with the derived key, and hand back the vault key only once it has opened.
|
|
13965
|
+
*
|
|
13966
|
+
* What is returned is the vault key, never the wrap key, as Device Approval caches: it keeps
|
|
13967
|
+
* working after the Recovery Code is regenerated on another device.
|
|
13968
|
+
*/
|
|
13969
|
+
/** The account has no vault yet, so there is nothing for a Recovery Code to open. */
|
|
13970
|
+
var NoVaultError = class extends Error {
|
|
13971
|
+
constructor() {
|
|
13972
|
+
super("This account has no encryption keys yet; there is nothing for a Recovery Code to open.");
|
|
13973
|
+
this.name = "NoVaultError";
|
|
13974
|
+
}
|
|
13975
|
+
};
|
|
13976
|
+
/**
|
|
13977
|
+
* @throws RecoveryCodeError when the code is malformed or does not open this account's vault.
|
|
13978
|
+
* @throws NoVaultError when the account has no vault.
|
|
13979
|
+
* Anything else (the vault could not be fetched) propagates unchanged: it says nothing about the code.
|
|
13980
|
+
*/
|
|
13981
|
+
async function openVaultWithRecoveryCode(api, code) {
|
|
13982
|
+
const wrapKey = await deriveVaultWrapKey(code);
|
|
13983
|
+
const stored = await api.getVault();
|
|
13984
|
+
if (!stored) throw new NoVaultError();
|
|
13985
|
+
try {
|
|
13986
|
+
return (await openVault(fromBase64Url(stored.vault), wrapKey)).vaultKey;
|
|
13987
|
+
} catch (error) {
|
|
13988
|
+
if (error instanceof EnvelopeError) throw new RecoveryCodeError("That Recovery Code does not open this account’s keys. Check it character by character; only the most recently issued code works.");
|
|
13989
|
+
throw error;
|
|
13990
|
+
}
|
|
13991
|
+
}
|
|
13992
|
+
//#endregion
|
|
12679
13993
|
//#region ../client/src/lib/sync/device-approval.ts
|
|
12680
13994
|
/**
|
|
12681
13995
|
* Device approval (ADR 0026 flows): unlock a NEW device from an already-unlocked one, so
|
|
@@ -12747,18 +14061,66 @@ var ApprovalAbandoned = class extends Error {
|
|
|
12747
14061
|
/** Approval rows expire server-side after ten minutes; poll a little longer and then give up. */
|
|
12748
14062
|
var APPROVAL_TIMEOUT_MS = 11 * 6e4;
|
|
12749
14063
|
var APPROVAL_POLL_MS = 2e3;
|
|
14064
|
+
/** How long a quit waits for the server to cancel the approval before leaving anyway. */
|
|
14065
|
+
var CANCEL_TIMEOUT_MS = 5e3;
|
|
14066
|
+
/** Shell exit codes for the signals that end a login: 128 plus the signal's number. */
|
|
14067
|
+
var SIGNAL_EXIT_CODES = {
|
|
14068
|
+
SIGHUP: 129,
|
|
14069
|
+
SIGINT: 130,
|
|
14070
|
+
SIGTERM: 143
|
|
14071
|
+
};
|
|
14072
|
+
/**
|
|
14073
|
+
* How the approval wait ends early. `r` switches to the Recovery Code; Ctrl-C (a key while the
|
|
14074
|
+
* terminal is in raw mode) and SIGINT, SIGTERM or SIGHUP quit. Either way the wait aborts, so it
|
|
14075
|
+
* cancels its approval server-side before anything exits: a login left pending showed its stale
|
|
14076
|
+
* code in every unlocked tab for ten minutes. `exitCode` is set once the user has quit.
|
|
14077
|
+
*/
|
|
14078
|
+
function approvalWaitControls() {
|
|
14079
|
+
const abort = new AbortController();
|
|
14080
|
+
let exitCode = null;
|
|
14081
|
+
return {
|
|
14082
|
+
signal: abort.signal,
|
|
14083
|
+
get exitCode() {
|
|
14084
|
+
return exitCode;
|
|
14085
|
+
},
|
|
14086
|
+
onKey(key) {
|
|
14087
|
+
if (key === "r" || key === "R") abort.abort();
|
|
14088
|
+
if (key === "") {
|
|
14089
|
+
exitCode = SIGNAL_EXIT_CODES.SIGINT;
|
|
14090
|
+
abort.abort();
|
|
14091
|
+
}
|
|
14092
|
+
},
|
|
14093
|
+
onSignal(name) {
|
|
14094
|
+
exitCode = SIGNAL_EXIT_CODES[name];
|
|
14095
|
+
abort.abort();
|
|
14096
|
+
}
|
|
14097
|
+
};
|
|
14098
|
+
}
|
|
14099
|
+
/** A sleep that ends as soon as the signal aborts, so a quit does not wait out the poll interval. */
|
|
14100
|
+
function sleepUnlessAborted(ms, signal) {
|
|
14101
|
+
return new Promise((resolve) => {
|
|
14102
|
+
if (signal.aborted) return resolve();
|
|
14103
|
+
const done = () => {
|
|
14104
|
+
clearTimeout(timer);
|
|
14105
|
+
signal.removeEventListener("abort", done);
|
|
14106
|
+
resolve();
|
|
14107
|
+
};
|
|
14108
|
+
const timer = setTimeout(done, ms);
|
|
14109
|
+
signal.addEventListener("abort", done, { once: true });
|
|
14110
|
+
});
|
|
14111
|
+
}
|
|
12750
14112
|
async function unlockByDeviceApproval(api, io) {
|
|
12751
14113
|
const request = await beginDeviceApproval(api);
|
|
12752
14114
|
const where = io.clientUrl ? `open EtherPK at ${io.clientUrl}` : "open EtherPK in a browser";
|
|
12753
14115
|
io.say("");
|
|
12754
|
-
io.say(`To approve this device, ${where} (any page - it need not be a note)
|
|
12755
|
-
io.say("with its
|
|
14116
|
+
io.say(`To approve this device, ${where} (any page - it need not be a note), connected to this account`);
|
|
14117
|
+
io.say("with its keys unlocked. A prompt will show a code; confirm it matches this one:");
|
|
12756
14118
|
io.say("");
|
|
12757
14119
|
io.say(` ${request.sas}`);
|
|
12758
14120
|
io.say("");
|
|
12759
14121
|
io.say("Waiting (up to ten minutes)…");
|
|
12760
14122
|
const abandon = async () => {
|
|
12761
|
-
await api.cancelDeviceApproval(request.id).catch(() => {});
|
|
14123
|
+
await Promise.race([api.cancelDeviceApproval(request.id).catch(() => {}), new Promise((resolve) => setTimeout(resolve, CANCEL_TIMEOUT_MS).unref())]);
|
|
12762
14124
|
throw new ApprovalAbandoned();
|
|
12763
14125
|
};
|
|
12764
14126
|
const deadline = Date.now() + APPROVAL_TIMEOUT_MS;
|
|
@@ -12773,12 +14135,12 @@ async function unlockByDeviceApproval(api, io) {
|
|
|
12773
14135
|
}
|
|
12774
14136
|
throw new Error("The approval was not confirmed in time. Run login again.");
|
|
12775
14137
|
}
|
|
12776
|
-
/**
|
|
14138
|
+
/**
|
|
14139
|
+
* Recovery Code → wrap key → open the vault; what comes back is the vault key to cache. The same
|
|
14140
|
+
* check the browser makes (recovery-unlock.ts): a wrong code is refused as wrong and nothing is cached.
|
|
14141
|
+
*/
|
|
12777
14142
|
async function unlockByRecoveryCode(api, code) {
|
|
12778
|
-
|
|
12779
|
-
const stored = await api.getVault();
|
|
12780
|
-
if (!stored) throw new Error("This account has no encryption keys yet; there is nothing for a Recovery Code to open.");
|
|
12781
|
-
return (await openVault(fromBase64Url(stored.vault), wrapKey)).vaultKey;
|
|
14143
|
+
return openVaultWithRecoveryCode(api, code);
|
|
12782
14144
|
}
|
|
12783
14145
|
//#endregion
|
|
12784
14146
|
//#region ../client/src/lib/document/publish/selection.ts
|
|
@@ -12898,7 +14260,11 @@ function publicDocumentsInNoPublication(documents, publications) {
|
|
|
12898
14260
|
}
|
|
12899
14261
|
//#endregion
|
|
12900
14262
|
//#region ../client/src/lib/document/publish/host/site-writer.ts
|
|
12901
|
-
/**
|
|
14263
|
+
/**
|
|
14264
|
+
* Stale files are removed only from the places the publisher owns. `etherpk-publish.json` is not
|
|
14265
|
+
* produced but stays owned, so the first publish after the upgrade deletes the copy an older
|
|
14266
|
+
* build left in the folder.
|
|
14267
|
+
*/
|
|
12902
14268
|
function isOwnedPath(path) {
|
|
12903
14269
|
if (path.startsWith("assets/") || path.startsWith("theme/")) return true;
|
|
12904
14270
|
if (path.includes("/")) return false;
|
|
@@ -13151,134 +14517,43 @@ function bundledThemeList() {
|
|
|
13151
14517
|
//#region ../client/src/lib/document/frontmatter/patch.ts
|
|
13152
14518
|
/**
|
|
13153
14519
|
* Rewriting arbitrary keys of a document's [[Frontmatter]] block, the way `withFrontmatterIdentity`
|
|
13154
|
-
* rewrites identity: other keys keep their values and
|
|
13155
|
-
* nothing would change, and a block whose YAML does not parse is left alone rather than
|
|
14520
|
+
* rewrites identity: other keys keep their values, order and comments, the same string comes back
|
|
14521
|
+
* when nothing would change, and a block whose YAML does not parse is left alone rather than
|
|
13156
14522
|
* destroyed. `null` removes a key. A block emptied of every key is removed with it. Identity keys
|
|
13157
14523
|
* are not this function's business: pass them through `withFrontmatterIdentity` so the registry
|
|
13158
14524
|
* rules (ADR 0061) apply.
|
|
13159
14525
|
*/
|
|
13160
|
-
function parseBlock$1(yaml) {
|
|
13161
|
-
try {
|
|
13162
|
-
const parsed = parse(yaml);
|
|
13163
|
-
if (parsed === null || parsed === void 0) return {};
|
|
13164
|
-
if (typeof parsed !== "object" || Array.isArray(parsed)) return null;
|
|
13165
|
-
return parsed;
|
|
13166
|
-
} catch {
|
|
13167
|
-
return null;
|
|
13168
|
-
}
|
|
13169
|
-
}
|
|
13170
|
-
function same(a, b) {
|
|
13171
|
-
return JSON.stringify(a) === JSON.stringify(b);
|
|
13172
|
-
}
|
|
13173
14526
|
function withFrontmatterPatch(text, patch, options = {}) {
|
|
13174
|
-
|
|
13175
|
-
|
|
13176
|
-
|
|
13177
|
-
|
|
13178
|
-
for (const [key, value] of Object.entries(patch)) if (value !== null && value !== void 0) data[key] = value;
|
|
13179
|
-
if (Object.keys(data).length === 0) return text;
|
|
13180
|
-
return `---\n${stringify(data)}---\n${text}`;
|
|
13181
|
-
}
|
|
13182
|
-
const data = parseBlock$1(span.body);
|
|
13183
|
-
if (data === null) return text;
|
|
13184
|
-
const next = {};
|
|
13185
|
-
let changed = false;
|
|
13186
|
-
for (const [key, value] of Object.entries(data)) {
|
|
13187
|
-
if (key in patch) {
|
|
13188
|
-
const wanted = patch[key];
|
|
13189
|
-
if (wanted === null || wanted === void 0) {
|
|
13190
|
-
changed = true;
|
|
13191
|
-
continue;
|
|
13192
|
-
}
|
|
13193
|
-
next[key] = wanted;
|
|
13194
|
-
if (!same(wanted, value)) changed = true;
|
|
13195
|
-
continue;
|
|
13196
|
-
}
|
|
13197
|
-
next[key] = value;
|
|
13198
|
-
}
|
|
13199
|
-
for (const [key, value] of Object.entries(patch)) {
|
|
13200
|
-
if (key in data || value === null || value === void 0) continue;
|
|
13201
|
-
next[key] = value;
|
|
13202
|
-
changed = true;
|
|
13203
|
-
}
|
|
13204
|
-
if (!changed) return text;
|
|
13205
|
-
if (Object.keys(next).length === 0) return text.slice(span.end);
|
|
13206
|
-
return `---\n${stringify(next)}---\n${text.slice(span.end)}`;
|
|
14527
|
+
return editFrontmatter(text, (block) => {
|
|
14528
|
+
for (const [key, value] of Object.entries(patch)) if (value === null || value === void 0) block.delete(key);
|
|
14529
|
+
else block.set(key, value);
|
|
14530
|
+
}, options);
|
|
13207
14531
|
}
|
|
13208
14532
|
//#endregion
|
|
13209
14533
|
//#region ../client/src/lib/document/frontmatter/publishing.ts
|
|
13210
14534
|
/**
|
|
13211
14535
|
* The publishing keys of a document's [[Frontmatter]] (ADR 0082): `public: true` is the consent
|
|
13212
|
-
* switch, `publications
|
|
13213
|
-
* key
|
|
13214
|
-
* key is that prompt to whoever edits the file by hand.
|
|
13215
|
-
*
|
|
13216
|
-
*
|
|
13217
|
-
*
|
|
13218
|
-
|
|
13219
|
-
|
|
13220
|
-
|
|
13221
|
-
|
|
13222
|
-
if (parsed === null || parsed === void 0) return {};
|
|
13223
|
-
if (typeof parsed !== "object" || Array.isArray(parsed)) return null;
|
|
13224
|
-
return parsed;
|
|
13225
|
-
} catch {
|
|
13226
|
-
return null;
|
|
13227
|
-
}
|
|
13228
|
-
}
|
|
13229
|
-
function sameList(a, b) {
|
|
13230
|
-
return Array.isArray(a) && a.length === b.length && a.every((v, i) => v === b[i]);
|
|
13231
|
-
}
|
|
14536
|
+
* switch, and `publications`, a list of publication ids, routes. A public document with no
|
|
14537
|
+
* publication keeps the key, written bare (`publications:`): public with nowhere to go is a state
|
|
14538
|
+
* worth a prompt, and the empty key is that prompt to whoever edits the file by hand. Bare rather
|
|
14539
|
+
* than `[]` because that is how the EtherPK style writes every empty value (ADR 0108), and because
|
|
14540
|
+
* Enter after a bare key opens the next line one level in, ready for the first `- id`. Not public, the empty key is
|
|
14541
|
+
* removed. Rewritten the way `withFrontmatterIdentity` rewrites identity: other keys keep their
|
|
14542
|
+
* values and order, the same string comes back when nothing would change, and a block whose YAML
|
|
14543
|
+
* does not parse is left alone rather than destroyed. Read through `readMembership` in
|
|
14544
|
+
* `publish/publication.ts`, which reads a bare key and `[]` alike; this file only writes.
|
|
14545
|
+
*/
|
|
13232
14546
|
/** The text with its publishing keys rewritten. `addBlock` adds a block to a document without one. */
|
|
13233
14547
|
function withPublishing(text, patch, options = {}) {
|
|
13234
14548
|
const publications = patch.publications === void 0 || patch.publications === null ? patch.publications : [...new Set(patch.publications.filter(isPublicationId))];
|
|
13235
|
-
|
|
13236
|
-
|
|
13237
|
-
if (
|
|
13238
|
-
|
|
13239
|
-
|
|
13240
|
-
if (publications
|
|
13241
|
-
|
|
13242
|
-
|
|
13243
|
-
}
|
|
13244
|
-
const data = parseBlock(span.body);
|
|
13245
|
-
if (data === null) return text;
|
|
13246
|
-
const next = {};
|
|
13247
|
-
let changed = false;
|
|
13248
|
-
const wantPublic = patch.public === void 0 ? data.public : patch.public;
|
|
13249
|
-
const keepEmpty = wantPublic === true;
|
|
13250
|
-
const wantPublications = publications === void 0 ? data.publications : publications && (publications.length > 0 || keepEmpty) ? publications : null;
|
|
13251
|
-
for (const [key, value] of Object.entries(data)) {
|
|
13252
|
-
if (key === "public") {
|
|
13253
|
-
if (wantPublic === null || wantPublic === void 0) changed = true;
|
|
13254
|
-
else {
|
|
13255
|
-
next.public = wantPublic;
|
|
13256
|
-
if (wantPublic !== value) changed = true;
|
|
13257
|
-
}
|
|
13258
|
-
continue;
|
|
13259
|
-
}
|
|
13260
|
-
if (key === "publications") {
|
|
13261
|
-
if (wantPublications === null || wantPublications === void 0) changed = true;
|
|
13262
|
-
else {
|
|
13263
|
-
next.publications = wantPublications;
|
|
13264
|
-
if (!sameList(value, wantPublications)) changed = true;
|
|
13265
|
-
}
|
|
13266
|
-
continue;
|
|
13267
|
-
}
|
|
13268
|
-
next[key] = value;
|
|
13269
|
-
}
|
|
13270
|
-
if (!("public" in data) && (wantPublic === true || wantPublic === false)) {
|
|
13271
|
-
next.public = wantPublic;
|
|
13272
|
-
changed = true;
|
|
13273
|
-
}
|
|
13274
|
-
if (!("publications" in data) && Array.isArray(wantPublications) && (wantPublications.length > 0 || keepEmpty)) {
|
|
13275
|
-
next.publications = wantPublications;
|
|
13276
|
-
changed = true;
|
|
13277
|
-
}
|
|
13278
|
-
if (!changed) return text;
|
|
13279
|
-
const yaml = Object.keys(next).length === 0 ? "" : stringify(next);
|
|
13280
|
-
if (yaml === "") return text.slice(span.end);
|
|
13281
|
-
return `---\n${yaml}---\n${text.slice(span.end)}`;
|
|
14549
|
+
return editFrontmatter(text, (block) => {
|
|
14550
|
+
if (patch.public === null) block.delete("public");
|
|
14551
|
+
else if (patch.public !== void 0) block.set("public", patch.public);
|
|
14552
|
+
if (publications === void 0) return;
|
|
14553
|
+
const keepEmpty = (patch.public === void 0 ? block.get("public") : patch.public) === true;
|
|
14554
|
+
if (publications === null || publications.length === 0 && !keepEmpty) block.delete("publications");
|
|
14555
|
+
else block.set("publications", publications.length === 0 ? null : publications);
|
|
14556
|
+
}, options);
|
|
13282
14557
|
}
|
|
13283
14558
|
//#endregion
|
|
13284
14559
|
//#region ../client/src/lib/document/publish/seeded.ts
|
|
@@ -13559,6 +14834,50 @@ function archiveHtml(journals) {
|
|
|
13559
14834
|
return `<ul class="journal-archive">${journals.map((j) => `<li><time datetime="${escapeHtml$6(j.date ?? "")}">${escapeHtml$6(j.date ?? "")}</time> <a href="${escapeHtml$6(j.url)}">${escapeHtml$6(j.title)}</a>${j.excerpt ? `<p>${escapeHtml$6(j.excerpt)}</p>` : ""}</li>`).join("")}</ul>`;
|
|
13560
14835
|
}
|
|
13561
14836
|
//#endregion
|
|
14837
|
+
//#region ../client/src/lib/document/publish/diagram-id.ts
|
|
14838
|
+
/**
|
|
14839
|
+
* Renaming a drawn Mermaid diagram's id, so a published page can carry it.
|
|
14840
|
+
*
|
|
14841
|
+
* Mermaid renders a diagram under the id it is given and scopes everything else to that id: every
|
|
14842
|
+
* rule of its inline `<style>` (`#gk-mermaid-4 .node rect { fill: … }`), its marker definitions
|
|
14843
|
+
* (`gk-mermaid-4_flowchart-pointEnd`) and the `url(#…)` references to them, and its accessible
|
|
14844
|
+
* title and description ids. The published `<svg>` must keep an id its styles match; without one
|
|
14845
|
+
* no rule applies and every node rect takes SVG's default black fill.
|
|
14846
|
+
*
|
|
14847
|
+
* The host's own id is not good enough to keep as it is. It comes from a render counter, so it
|
|
14848
|
+
* changes from one publish to the next (and the browser host reuses the editor's cached drawings),
|
|
14849
|
+
* which would rewrite every page with a diagram on every publish; and one drawing is reused for
|
|
14850
|
+
* every occurrence of the same source, so a diagram shown twice on a page would put one id there
|
|
14851
|
+
* twice. The publisher therefore renames each occurrence to an id of its own choosing.
|
|
14852
|
+
*/
|
|
14853
|
+
/** The root element's id, or null. Only the opening tag is read. */
|
|
14854
|
+
function rootId(svg) {
|
|
14855
|
+
const open = /^\s*<svg\b[^>]*>/.exec(svg)?.[0];
|
|
14856
|
+
if (!open) return null;
|
|
14857
|
+
return /\sid="([^"]+)"/.exec(open)?.[1] ?? null;
|
|
14858
|
+
}
|
|
14859
|
+
/** The id the inline styles are scoped to, for a drawing whose root has lost its id. */
|
|
14860
|
+
function styleScope(svg) {
|
|
14861
|
+
const style = /<style\b[^>]*>([\s\S]*?)<\/style>/.exec(svg)?.[1];
|
|
14862
|
+
if (!style) return null;
|
|
14863
|
+
return /#([A-Za-z][\w-]*)(?=[\s{.,>:])/.exec(style)?.[1] ?? null;
|
|
14864
|
+
}
|
|
14865
|
+
function escapeRegExp(text) {
|
|
14866
|
+
return text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
14867
|
+
}
|
|
14868
|
+
/**
|
|
14869
|
+
* The drawing with its id, and everything scoped to it, renamed to `id`. `id` must be a valid
|
|
14870
|
+
* CSS identifier that starts with a letter. A drawing with no id anywhere is returned as it is:
|
|
14871
|
+
* nothing in it is scoped, so there is nothing to keep in step.
|
|
14872
|
+
*/
|
|
14873
|
+
function withDiagramId(svg, id) {
|
|
14874
|
+
const current = rootId(svg);
|
|
14875
|
+
if (current) return svg.replace(new RegExp(`${escapeRegExp(current)}(?![A-Za-z0-9])`, "g"), id);
|
|
14876
|
+
const scope = styleScope(svg);
|
|
14877
|
+
if (!scope) return svg;
|
|
14878
|
+
return svg.replace(new RegExp(`${escapeRegExp(scope)}(?![A-Za-z0-9])`, "g"), id).replace(/^(\s*<svg\b)/, `$1 id="${id}"`);
|
|
14879
|
+
}
|
|
14880
|
+
//#endregion
|
|
13562
14881
|
//#region ../client/src/lib/document/code-languages.ts
|
|
13563
14882
|
/**
|
|
13564
14883
|
* The grammar for a [[Fenced Code Block]]'s info-string, from the registry the editor nests inside
|
|
@@ -14329,7 +15648,7 @@ function allocate(documents) {
|
|
|
14329
15648
|
const derived = [];
|
|
14330
15649
|
for (const doc of documents) {
|
|
14331
15650
|
const value = parseFrontmatter(doc.text).data.slug;
|
|
14332
|
-
if (value
|
|
15651
|
+
if (isEmptyValue(value)) {
|
|
14333
15652
|
derived.push(doc);
|
|
14334
15653
|
continue;
|
|
14335
15654
|
}
|
|
@@ -14416,7 +15735,6 @@ function createThemeRenderer(theme, includes) {
|
|
|
14416
15735
|
* a pre-render pass for the asynchronous bits (diagrams, code); then rendering, includes, the
|
|
14417
15736
|
* theme, the derived files. Everything derived is computed from the included documents alone.
|
|
14418
15737
|
*/
|
|
14419
|
-
var DATE = /^\d{4}-\d{2}-\d{2}$/;
|
|
14420
15738
|
function assetHrefOf(name) {
|
|
14421
15739
|
return `assets/${encodeURIComponent(name)}`;
|
|
14422
15740
|
}
|
|
@@ -14622,18 +15940,32 @@ async function publishPublication(source, publication, env, options = {}) {
|
|
|
14622
15940
|
const html = await highlight(fence.lang, fence.code);
|
|
14623
15941
|
if (html !== null) highlighted.set(key, html);
|
|
14624
15942
|
}
|
|
15943
|
+
let diagramScope = {
|
|
15944
|
+
prefix: "mermaid",
|
|
15945
|
+
n: 0
|
|
15946
|
+
};
|
|
15947
|
+
const scopeDiagrams = (prefix) => {
|
|
15948
|
+
diagramScope = {
|
|
15949
|
+
prefix: prefix.replace(/[^A-Za-z0-9_-]/g, "-"),
|
|
15950
|
+
n: 0
|
|
15951
|
+
};
|
|
15952
|
+
};
|
|
14625
15953
|
const renderer = createDocumentRenderer({
|
|
14626
15954
|
resolve: resolver.resolve,
|
|
14627
15955
|
assetHref: assetHrefOf,
|
|
14628
|
-
mermaidSvg: (src) =>
|
|
15956
|
+
mermaidSvg: (src) => {
|
|
15957
|
+
const svg = mermaidSvg.get(src);
|
|
15958
|
+
return svg === void 0 ? void 0 : withDiagramId(svg, `${diagramScope.prefix}_${++diagramScope.n}`);
|
|
15959
|
+
},
|
|
14629
15960
|
highlighted: (lang, code) => highlighted.get(`${lang}\n${code}`)
|
|
14630
15961
|
});
|
|
14631
15962
|
const pages = [];
|
|
14632
15963
|
let done = 0;
|
|
14633
15964
|
for (const doc of included) {
|
|
14634
15965
|
progress("rendering", done++, included.length);
|
|
14635
|
-
const rendered = renderer.render(bodyOf(doc));
|
|
14636
15966
|
const slug = slugs.get(conceptKey$1(doc.concept));
|
|
15967
|
+
scopeDiagrams(`mermaid_${slug}`);
|
|
15968
|
+
const rendered = renderer.render(bodyOf(doc));
|
|
14637
15969
|
const isHome = doc === homeDoc;
|
|
14638
15970
|
const page = {
|
|
14639
15971
|
doc,
|
|
@@ -14646,17 +15978,9 @@ async function publishPublication(source, publication, env, options = {}) {
|
|
|
14646
15978
|
};
|
|
14647
15979
|
if (doc.kind === "journal") page.date = doc.concept;
|
|
14648
15980
|
else {
|
|
14649
|
-
const
|
|
14650
|
-
if (date !== void 0)
|
|
14651
|
-
|
|
14652
|
-
if (DATE.test(text)) page.date = text;
|
|
14653
|
-
else warnings.push({
|
|
14654
|
-
level: "warning",
|
|
14655
|
-
code: "invalid-date",
|
|
14656
|
-
message: `"${doc.concept}" has \`date: ${String(date)}\`, which is not a calendar day (YYYY-MM-DD); the document is undated.`,
|
|
14657
|
-
concept: doc.concept
|
|
14658
|
-
});
|
|
14659
|
-
}
|
|
15981
|
+
const dated = readPageDate(doc);
|
|
15982
|
+
if (dated.date !== void 0) page.date = dated.date;
|
|
15983
|
+
take(dated.issues);
|
|
14660
15984
|
}
|
|
14661
15985
|
for (const link of linksOf(bodyOf(doc), resolver, doc)) report.missingLinks.push(link);
|
|
14662
15986
|
pages.push(page);
|
|
@@ -14711,7 +16035,10 @@ async function publishPublication(source, publication, env, options = {}) {
|
|
|
14711
16035
|
continue;
|
|
14712
16036
|
}
|
|
14713
16037
|
customCss += (customCss ? "\n" : "") + fence.code;
|
|
14714
|
-
} else
|
|
16038
|
+
} else {
|
|
16039
|
+
scopeDiagrams(`mermaid__${slot}`);
|
|
16040
|
+
includes.set(slot, renderer.render(body).html);
|
|
16041
|
+
}
|
|
14715
16042
|
report.includes.push({
|
|
14716
16043
|
name: slot,
|
|
14717
16044
|
source: "page",
|
|
@@ -15088,9 +16415,10 @@ function createThemeLoader(deps) {
|
|
|
15088
16415
|
/** Replace a document's frontmatter block with the one `rewrite` produces, through its live handle. */
|
|
15089
16416
|
async function rewriteFrontmatter(store, concept, rewrite) {
|
|
15090
16417
|
await store.whenReady?.(concept);
|
|
16418
|
+
await store.flushDocument?.(concept);
|
|
15091
16419
|
const handle = store.open(concept);
|
|
15092
16420
|
const text = handle.getText();
|
|
15093
|
-
const next = rewrite(text);
|
|
16421
|
+
const next = withAliasesInAddedBlock(text, rewrite(text), registryAliases(store, concept));
|
|
15094
16422
|
if (next === text) return false;
|
|
15095
16423
|
const before = frontmatterSpan(text)?.end ?? 0;
|
|
15096
16424
|
const after = frontmatterSpan(next)?.end ?? 0;
|
|
@@ -15102,6 +16430,11 @@ async function rewriteFrontmatter(store, concept, rewrite) {
|
|
|
15102
16430
|
await store.flushDocument?.(concept);
|
|
15103
16431
|
return true;
|
|
15104
16432
|
}
|
|
16433
|
+
/** The document's aliases as its store's registry holds them, found by its identity key. */
|
|
16434
|
+
function registryAliases(store, concept) {
|
|
16435
|
+
const key = conceptKey(concept);
|
|
16436
|
+
return store.listDocuments?.().find((entry) => entry.key === key)?.aliases ?? [];
|
|
16437
|
+
}
|
|
15105
16438
|
/** A publication id from a title: `Docs Site` → `docs-site`. */
|
|
15106
16439
|
function suggestPublicationId(title) {
|
|
15107
16440
|
return publishSlug(title);
|
|
@@ -15210,7 +16543,6 @@ async function runPublish(publication, deps) {
|
|
|
15210
16543
|
code: "documents-unsettled",
|
|
15211
16544
|
message: `${deps.unsettled.length} document${deps.unsettled.length === 1 ? " has" : "s have"} not finished syncing to this device and ${deps.unsettled.length === 1 ? "was" : "were"} left out: ${deps.unsettled.slice(0, 5).join(", ")}${deps.unsettled.length > 5 ? "…" : ""}. Publish again once sync has caught up.`
|
|
15212
16545
|
});
|
|
15213
|
-
if (report.ok) bundle.set("etherpk-publish.json", `${JSON.stringify(report, null, 2)}\n`);
|
|
15214
16546
|
return {
|
|
15215
16547
|
report,
|
|
15216
16548
|
bundle,
|
|
@@ -15355,7 +16687,6 @@ async function openDiagramRenderer(env) {
|
|
|
15355
16687
|
const { svg } = await m.render(renderId, text);
|
|
15356
16688
|
const el = new DOMParser().parseFromString(svg, "text/html").querySelector("svg");
|
|
15357
16689
|
if (!el) throw new Error("Mermaid produced no diagram.");
|
|
15358
|
-
el.removeAttribute("id");
|
|
15359
16690
|
el.setAttribute("role", "img");
|
|
15360
16691
|
return el.outerHTML;
|
|
15361
16692
|
} finally {
|
|
@@ -15862,7 +17193,7 @@ function agentFrontmatter(rawText) {
|
|
|
15862
17193
|
/**
|
|
15863
17194
|
* The block after a patch. `public` and `publications` go through the publishing writer, so its
|
|
15864
17195
|
* rules hold whoever writes them: an id is lower-case letters, digits and hyphens, and a public
|
|
15865
|
-
* document with an empty list keeps
|
|
17196
|
+
* document with an empty list keeps the key, written bare, as the prompt it is. Everything else is a
|
|
15866
17197
|
* plain key. An invalid publication id is refused rather than dropped, which is what the
|
|
15867
17198
|
* writer would do: an agent that misspells an id must hear about it.
|
|
15868
17199
|
*/
|
|
@@ -15955,7 +17286,7 @@ function bounded(value, fallback, max) {
|
|
|
15955
17286
|
*/
|
|
15956
17287
|
async function settle$2(graph) {
|
|
15957
17288
|
const result = await graph.settle();
|
|
15958
|
-
if (!result.settled) throw new ToolError("not_settled", result.message);
|
|
17289
|
+
if (!result.settled) throw new ToolError(result.code ?? "not_settled", result.message);
|
|
15959
17290
|
}
|
|
15960
17291
|
async function listDocuments(graph, args = {}) {
|
|
15961
17292
|
await graph.store.refresh();
|
|
@@ -16045,7 +17376,24 @@ function round(similarity) {
|
|
|
16045
17376
|
return Math.round(similarity * 1e3) / 1e3;
|
|
16046
17377
|
}
|
|
16047
17378
|
async function searchText(graph, query, offset, limit) {
|
|
16048
|
-
const
|
|
17379
|
+
const keys = await graph.index.propertyKeys();
|
|
17380
|
+
const { words, filters } = parseSearchQuery(query, new Set(keys.map((info) => info.key.toLowerCase())));
|
|
17381
|
+
if (filters.length > 0 && words.trim() === "") {
|
|
17382
|
+
const documents = await graph.index.propertyMatch(filters);
|
|
17383
|
+
return {
|
|
17384
|
+
results: documents.slice(offset, offset + limit).map((doc) => ({
|
|
17385
|
+
concept: doc.concept,
|
|
17386
|
+
kind: doc.kind,
|
|
17387
|
+
matches: 0,
|
|
17388
|
+
hits: [],
|
|
17389
|
+
properties: doc.properties
|
|
17390
|
+
})),
|
|
17391
|
+
total: documents.length,
|
|
17392
|
+
totalCapped: false,
|
|
17393
|
+
hasMore: offset + limit < documents.length
|
|
17394
|
+
};
|
|
17395
|
+
}
|
|
17396
|
+
const [result, count] = await Promise.all([graph.index.searchText(words, offset, limit, filters), graph.index.searchTextCount(words, filters)]);
|
|
16049
17397
|
return {
|
|
16050
17398
|
results: result.groups.map((group) => ({
|
|
16051
17399
|
concept: group.concept,
|
|
@@ -16195,7 +17543,7 @@ async function setFrontmatter(graph, args) {
|
|
|
16195
17543
|
const body = await liveText(graph, identity);
|
|
16196
17544
|
refuseIfProtected(identity.concept, body);
|
|
16197
17545
|
const raw = graph.store.openRaw(identity.concept).getText();
|
|
16198
|
-
const next = patchedText(raw, args.patch);
|
|
17546
|
+
const next = withAliasesInAddedBlock(raw, patchedText(raw, args.patch), identity.aliases);
|
|
16199
17547
|
applyBlock(graph, identity.concept, raw, next);
|
|
16200
17548
|
await settle$2(graph);
|
|
16201
17549
|
return {
|
|
@@ -16669,9 +18017,30 @@ async function graphInfo(graph) {
|
|
|
16669
18017
|
function hostOf$1(host) {
|
|
16670
18018
|
return {
|
|
16671
18019
|
env: host?.env ?? process.env,
|
|
16672
|
-
cmd: host?.cmd ?? "etherpk-mcp"
|
|
18020
|
+
cmd: host?.cmd ?? "etherpk-mcp",
|
|
18021
|
+
via: host?.via ?? "agent"
|
|
16673
18022
|
};
|
|
16674
18023
|
}
|
|
18024
|
+
/**
|
|
18025
|
+
* The refusal for an id no publication has, naming the ids there are. An agent is pointed at
|
|
18026
|
+
* its tool; a person at the command line, who has no agent tool to run, at the Publish tab.
|
|
18027
|
+
*/
|
|
18028
|
+
function publicationNotFound(id, publications, via) {
|
|
18029
|
+
const where = via === "cli" ? "Settings → Publish in EtherPK lists them." : "list_publications shows them.";
|
|
18030
|
+
return new ToolError("publication_not_found", `No publication has the id "${id}". ${publications.length === 0 ? "This graph defines no publication yet." : `Its publications: ${publications.map((p) => p.id).join(", ")}.`} ${where}`);
|
|
18031
|
+
}
|
|
18032
|
+
/**
|
|
18033
|
+
* The publication with this id, or the refusal a publish would give. The command line asks
|
|
18034
|
+
* before it remembers a publish folder, so a mistyped id is not written down.
|
|
18035
|
+
*/
|
|
18036
|
+
async function findPublication(graph, id, host) {
|
|
18037
|
+
const { via } = hostOf$1(host);
|
|
18038
|
+
await graph.store.refresh();
|
|
18039
|
+
const { publications } = summarisePublishing((await graph.publishing.readSource()).source);
|
|
18040
|
+
const found = publications.find((p) => p.id === id.trim());
|
|
18041
|
+
if (!found) throw publicationNotFound(id, publications, via);
|
|
18042
|
+
return found;
|
|
18043
|
+
}
|
|
16675
18044
|
/** The publication page's mapping and body, through the raw handles: what `publish-service` writes to. */
|
|
16676
18045
|
function frontmatterStore(graph) {
|
|
16677
18046
|
return {
|
|
@@ -16772,8 +18141,9 @@ async function createPublication(graph, args, host) {
|
|
|
16772
18141
|
async function updatePublication(graph, args, host) {
|
|
16773
18142
|
const { env } = hostOf$1(host);
|
|
16774
18143
|
await graph.store.refresh();
|
|
16775
|
-
const
|
|
16776
|
-
|
|
18144
|
+
const all = summarisePublishing((await graph.publishing.readSource()).source).publications;
|
|
18145
|
+
const current = all.find((p) => p.id === args.id.trim());
|
|
18146
|
+
if (!current) throw publicationNotFound(args.id, all, hostOf$1(host).via);
|
|
16777
18147
|
const changes = args.changes ?? {};
|
|
16778
18148
|
if (changes.kind !== void 0 && changes.kind !== null && changes.kind !== "docs" && changes.kind !== "blog") throw new ToolError("invalid_argument", "kind must be \"docs\" or \"blog\".");
|
|
16779
18149
|
if (changes.selection !== void 0 && changes.selection !== null && changes.selection !== "named" && changes.selection !== "all-public") throw new ToolError("invalid_argument", "selection must be \"named\" or \"all-public\".");
|
|
@@ -16786,6 +18156,12 @@ async function updatePublication(graph, args, host) {
|
|
|
16786
18156
|
const folderOf = await foldersFor(graph, env);
|
|
16787
18157
|
return { publication: publicationView(after ?? current, folderOf(current.id)) };
|
|
16788
18158
|
}
|
|
18159
|
+
/** How many items fall under each key. */
|
|
18160
|
+
function countBy(items, keyOf) {
|
|
18161
|
+
const counts = {};
|
|
18162
|
+
for (const item of items) counts[keyOf(item)] = (counts[keyOf(item)] ?? 0) + 1;
|
|
18163
|
+
return counts;
|
|
18164
|
+
}
|
|
16789
18165
|
/** The first entries of a long list, and how many there were. */
|
|
16790
18166
|
function head(items, n = 20) {
|
|
16791
18167
|
return {
|
|
@@ -16796,15 +18172,22 @@ function head(items, n = 20) {
|
|
|
16796
18172
|
/**
|
|
16797
18173
|
* Publish one publication into its Publish Folder and report. The folder is the one a person
|
|
16798
18174
|
* set for this graph and publication on this machine (ADR 0086); a publish with Mermaid needs
|
|
16799
|
-
* the browser (ADR 0084).
|
|
18175
|
+
* the browser (ADR 0084).
|
|
18176
|
+
*
|
|
18177
|
+
* The result IS the report, trimmed to counts and first entries: nothing of it is written into
|
|
18178
|
+
* the folder, because a static host serves whatever the folder holds and the report names every
|
|
18179
|
+
* document the site leaves out, protected ones included. The documents left out are counted by
|
|
18180
|
+
* reason rather than listed: that is every document outside the publication, most of a large
|
|
18181
|
+
* graph. The command line prints {@link cliPublishOutput} instead, which names nothing the site
|
|
18182
|
+
* leaves out.
|
|
16800
18183
|
*/
|
|
16801
18184
|
async function publish(graph, args, host) {
|
|
16802
|
-
const { env, cmd } = hostOf$1(host);
|
|
18185
|
+
const { env, cmd, via } = hostOf$1(host);
|
|
16803
18186
|
await graph.store.refresh();
|
|
16804
18187
|
const { source, unsettled } = await graph.publishing.readSource();
|
|
16805
18188
|
const summary = summarisePublishing(source);
|
|
16806
18189
|
const publication = summary.publications.find((p) => p.id === args.id.trim());
|
|
16807
|
-
if (!publication) throw
|
|
18190
|
+
if (!publication) throw publicationNotFound(args.id, summary.publications, via);
|
|
16808
18191
|
const folder = (await foldersFor(graph, env))(publication.id);
|
|
16809
18192
|
if (!folder) {
|
|
16810
18193
|
const where = graph.backend.kind === "folder" ? `--folder "${graph.backend.path}"` : `--graph "${graph.name}"`;
|
|
@@ -16851,20 +18234,47 @@ async function publish(graph, args, host) {
|
|
|
16851
18234
|
},
|
|
16852
18235
|
errors: run.report.errors,
|
|
16853
18236
|
warnings: run.report.warnings,
|
|
16854
|
-
missingLinks:
|
|
18237
|
+
missingLinks: {
|
|
18238
|
+
...head(run.report.missingLinks),
|
|
18239
|
+
byStatus: countBy(run.report.missingLinks, (link) => link.status)
|
|
18240
|
+
},
|
|
16855
18241
|
assets: {
|
|
16856
18242
|
copied: run.report.assets.copied.length,
|
|
16857
18243
|
missing: run.report.assets.missing
|
|
16858
18244
|
},
|
|
16859
18245
|
collisions: run.report.collisions,
|
|
16860
18246
|
publicInNoPublication: run.report.publicInNoPublication,
|
|
16861
|
-
...unsettled.length > 0 ? { unsettled } : {}
|
|
16862
|
-
report: run.report.ok ? `${folder}/etherpk-publish.json` : null
|
|
18247
|
+
...unsettled.length > 0 ? { unsettled } : {}
|
|
16863
18248
|
};
|
|
16864
18249
|
} finally {
|
|
16865
18250
|
await renderer?.dispose();
|
|
16866
18251
|
}
|
|
16867
18252
|
}
|
|
18253
|
+
/**
|
|
18254
|
+
* What `etherpk-mcp publish` prints: the tool's result with every name the site leaves out
|
|
18255
|
+
* reduced to a count. A scheduled publish's log can be as public as the site (a public CI run),
|
|
18256
|
+
* so links are counted by status (a "private" status says a hidden page exists), documents not
|
|
18257
|
+
* yet synced and public documents in no publication are counted, and warnings keep their codes
|
|
18258
|
+
* without the sentences that name pages. What was published, and the errors that stopped a
|
|
18259
|
+
* publish, are printed as they are: the site shows the one, and the other is what to fix.
|
|
18260
|
+
*/
|
|
18261
|
+
function cliPublishOutput(result) {
|
|
18262
|
+
const { unsettled, ...rest } = result;
|
|
18263
|
+
return {
|
|
18264
|
+
...rest,
|
|
18265
|
+
missingLinks: {
|
|
18266
|
+
total: result.missingLinks.total,
|
|
18267
|
+
byStatus: result.missingLinks.byStatus
|
|
18268
|
+
},
|
|
18269
|
+
warnings: {
|
|
18270
|
+
total: result.warnings.length,
|
|
18271
|
+
byCode: countBy(result.warnings, (warning) => warning.code)
|
|
18272
|
+
},
|
|
18273
|
+
publicInNoPublication: result.publicInNoPublication.length,
|
|
18274
|
+
...unsettled ? { unsettled: unsettled.length } : {},
|
|
18275
|
+
note: "Documents the site leaves out are counted here, not named. Settings → Publish in EtherPK shows the full report."
|
|
18276
|
+
};
|
|
18277
|
+
}
|
|
16868
18278
|
/** The publishing half of `graph_info`: each publication with its folder here, and whether diagrams can be drawn. */
|
|
16869
18279
|
async function publishingInfo(graph, host) {
|
|
16870
18280
|
const { env, cmd } = hostOf$1(host);
|
|
@@ -17480,13 +18890,22 @@ function failed(error) {
|
|
|
17480
18890
|
isError: true
|
|
17481
18891
|
};
|
|
17482
18892
|
}
|
|
17483
|
-
async function
|
|
18893
|
+
async function runTool(work) {
|
|
17484
18894
|
try {
|
|
17485
18895
|
return ok(await work());
|
|
17486
18896
|
} catch (error) {
|
|
17487
18897
|
return failed(error);
|
|
17488
18898
|
}
|
|
17489
18899
|
}
|
|
18900
|
+
/**
|
|
18901
|
+
* The refusal every tool gives once the Sync Server has ended access, instead of answering from
|
|
18902
|
+
* a cache that can no longer be trusted or saved.
|
|
18903
|
+
*/
|
|
18904
|
+
function accessEndedError(graph, cmd) {
|
|
18905
|
+
const loss = graph.accessLoss();
|
|
18906
|
+
if (!loss) return null;
|
|
18907
|
+
return loss.kind === "membership" ? new ToolError("access_removed", `This account no longer has access to the graph: it left, was removed by the owner, or the graph was deleted. Run ${cmd} graphs to see the graphs it can reach.`) : new ToolError("token_revoked", `The Sync Server no longer accepts this computer's access token: it was revoked or has expired, or the account's password was reset. Run ${cmd} login again.`);
|
|
18908
|
+
}
|
|
17490
18909
|
var concept = z.string().min(1).describe("A page title, one of its aliases, a journal day as YYYY-MM-DD, or \"today\".");
|
|
17491
18910
|
var offset = z.number().int().nonnegative().optional().describe("Skip this many results (paging).");
|
|
17492
18911
|
/** A frontmatter value as JSON carries it; the writer turns it into YAML. */
|
|
@@ -17499,6 +18918,11 @@ var frontmatterValue = z.union([
|
|
|
17499
18918
|
z.record(z.string(), z.unknown())
|
|
17500
18919
|
]);
|
|
17501
18920
|
function createMcpServer(graph, info) {
|
|
18921
|
+
const run = (work) => runTool(() => {
|
|
18922
|
+
const ended = accessEndedError(graph, info.cmd ?? "etherpk-mcp");
|
|
18923
|
+
if (ended) throw ended;
|
|
18924
|
+
return work();
|
|
18925
|
+
});
|
|
17502
18926
|
const server = new McpServer({
|
|
17503
18927
|
name: "etherpk",
|
|
17504
18928
|
version: info.version
|
|
@@ -17539,7 +18963,7 @@ function createMcpServer(graph, info) {
|
|
|
17539
18963
|
}, async (args) => run(() => readDocuments(graph, args)));
|
|
17540
18964
|
server.registerTool("search", {
|
|
17541
18965
|
title: "Search",
|
|
17542
|
-
description: `Search every unprotected document. mode "text" (default): words match by prefix, "quoted phrases" match exactly; documents come most-matches-first with the matching lines. mode "semantic": the passages closest in MEANING to the query, best first, each with its text, a similarity 0..1 and where it is (concept, breadcrumb of headings and parent bullets, start and end line) - finds a note that says "self-assessment is due 31 January" for "tax deadline". Line numbers are 0-based in every mode. Passage text is the note's own words with bullet markers and [[ ]] stripped: quote and cite it, but anchor edit_document on text from read_document. Use semantic for questions and descriptions, text for names, identifiers and exact phrases; mode "hybrid" returns both as separate groups. Semantic results carry complete: false while this computer is still embedding the graph (rerun later for more). If semantic mode answers error semantic_unavailable, it is not set up on this computer: tell the user the command in the message. At most 50 documents per call.`,
|
|
18966
|
+
description: `Search every unprotected document. mode "text" (default): words match by prefix, "quoted phrases" match exactly; documents come most-matches-first with the matching lines. In text mode a term key:value is a property filter on the documents' frontmatter when some document has that key (keys and values ignore case; key:"two words", -key:value excludes, key:* means the key is set, key:va* matches the start of a value, publication.id:docs reaches a nested key, title: and aliases: match the document's names); filters alone list every matching document with the values they matched and no hits. mode "semantic": the passages closest in MEANING to the query, best first, each with its text, a similarity 0..1 and where it is (concept, breadcrumb of headings and parent bullets, start and end line) - finds a note that says "self-assessment is due 31 January" for "tax deadline". Line numbers are 0-based in every mode. Passage text is the note's own words with bullet markers and [[ ]] stripped: quote and cite it, but anchor edit_document on text from read_document. Use semantic for questions and descriptions, text for names, identifiers and exact phrases; mode "hybrid" returns both as separate groups. Semantic results carry complete: false while this computer is still embedding the graph (rerun later for more). If semantic mode answers error semantic_unavailable, it is not set up on this computer: tell the user the command in the message. At most 50 documents per call.`,
|
|
17543
18967
|
inputSchema: {
|
|
17544
18968
|
query: z.string().min(1),
|
|
17545
18969
|
mode: z.enum([
|
|
@@ -17731,7 +19155,7 @@ function createMcpServer(graph, info) {
|
|
|
17731
19155
|
}, async (args) => run(() => updatePublication(graph, args, host)));
|
|
17732
19156
|
server.registerTool("publish", {
|
|
17733
19157
|
title: "Publish",
|
|
17734
|
-
description: "Render a publication to its publish folder on this machine and return the report: what was included and why documents were left out, missing links, assets, warnings. The folder is the one the user set with \"etherpk-mcp publish --publication <id> --out <dir>\" (error \"no_publish_folder\" until then; the tool never chooses a folder). Pages with Mermaid diagrams need the browser from \"diagrams setup\" (error \"chromium_unavailable\"). The
|
|
19158
|
+
description: "Render a publication to its publish folder on this machine and return the report: what was included and why documents were left out, missing links, assets, warnings. The folder is the one the user set with \"etherpk-mcp publish --publication <id> --out <dir>\" (error \"no_publish_folder\" until then; the tool never chooses a folder). Pages with Mermaid diagrams need the browser from \"diagrams setup\" (error \"chromium_unavailable\"). The report is this result, trimmed to counts and first entries; nothing of it is written into the folder, so the site never names the documents it leaves out. Publishing writes files; it does not deploy them.",
|
|
17735
19159
|
inputSchema: { id: z.string().min(1) }
|
|
17736
19160
|
}, async (args) => run(() => publish(graph, args, host)));
|
|
17737
19161
|
server.registerTool("list_themes", {
|
|
@@ -18013,7 +19437,7 @@ function bindServeLifetime(deps) {
|
|
|
18013
19437
|
* etherpk-mcp diagrams setup | status
|
|
18014
19438
|
*
|
|
18015
19439
|
* One config file holds a login per Sync Server (ADR 0075). `--sync-server` names the one a
|
|
18016
|
-
* command means and may be left out while only one
|
|
19440
|
+
* command means and may be left out while there is only one login.
|
|
18017
19441
|
*
|
|
18018
19442
|
* `serve` speaks MCP over stdio, so everything for the human goes to stderr; stdout belongs
|
|
18019
19443
|
* to the agent. `login` and `graphs` are interactive and print to stdout.
|
|
@@ -18032,11 +19456,11 @@ var USAGE = `etherpk-mcp - EtherPK Headless Client (an MCP server over one synce
|
|
|
18032
19456
|
Sign this machine in as a device of your account. Prompts for a Personal Access
|
|
18033
19457
|
Token (an account-wide one, from the Sync Server portal at <url>/account/tokens)
|
|
18034
19458
|
unless --pat or ETHERPK_PAT is given, then unlocks your keys by Device Approval:
|
|
18035
|
-
open EtherPK in a browser
|
|
19459
|
+
open EtherPK in a browser connected to the account with its keys unlocked and
|
|
18036
19460
|
confirm the code shown. Press r while waiting, or pass --recovery-code, to type
|
|
18037
19461
|
your Recovery Code instead (or ETHERPK_RECOVERY_CODE, for a scripted setup).
|
|
18038
19462
|
${CMD} graphs [--sync-server <url>]
|
|
18039
|
-
List the synced graphs each
|
|
19463
|
+
List the synced graphs each logged-in account can reach, by name and id.
|
|
18040
19464
|
${CMD} serve --graph <id or name> [--sync-server <url>] [--no-semantic]
|
|
18041
19465
|
Serve one synced graph to an agent over stdio. For Claude Code:
|
|
18042
19466
|
claude mcp add etherpk -- npx @appsoftwareltd/etherpk-mcp serve --sync-server <url> --graph <id>
|
|
@@ -18074,8 +19498,8 @@ var USAGE = `etherpk-mcp - EtherPK Headless Client (an MCP server over one synce
|
|
|
18074
19498
|
${CMD} diagrams status
|
|
18075
19499
|
Which browser a publish would use, if any.
|
|
18076
19500
|
|
|
18077
|
-
This machine can
|
|
18078
|
-
a command means, and can be left out while
|
|
19501
|
+
This machine can hold logins for several Sync Servers at once; --sync-server says which one
|
|
19502
|
+
a command means, and can be left out while there is only one. The config file is
|
|
18079
19503
|
${defaultConfigPath()} (override with
|
|
18080
19504
|
ETHERPK_MCP_CONFIG); cached graphs live under ~/.cache/etherpk/mcp (override with
|
|
18081
19505
|
ETHERPK_MCP_CACHE_DIR).
|
|
@@ -18085,8 +19509,12 @@ function fail(message) {
|
|
|
18085
19509
|
console.error(message);
|
|
18086
19510
|
process.exit(1);
|
|
18087
19511
|
}
|
|
18088
|
-
|
|
18089
|
-
|
|
19512
|
+
/**
|
|
19513
|
+
* Ask on the terminal; `hint` says how to give the answer when there is none (a script, or
|
|
19514
|
+
* an agent starting the process), since each question has its own option or variable.
|
|
19515
|
+
*/
|
|
19516
|
+
async function ask(question, { secret = false, hint }) {
|
|
19517
|
+
if (!process.stdin.isTTY) fail(`${question.replace(/:\s*$/, "")}: no terminal to ask on; ${hint}.`);
|
|
18090
19518
|
if (!secret) {
|
|
18091
19519
|
const rl = createInterface({
|
|
18092
19520
|
input: process.stdin,
|
|
@@ -18134,24 +19562,30 @@ function requireServer(config, wanted) {
|
|
|
18134
19562
|
if (selection.ok) return selection.credentials;
|
|
18135
19563
|
switch (selection.reason) {
|
|
18136
19564
|
case "none": return fail(`Not logged in on this machine. Run: ${CMD} login --sync-server <url>`);
|
|
18137
|
-
case "unknown": return fail(`Not logged in to ${selection.syncServer}.
|
|
18138
|
-
case "ambiguous": return fail(`
|
|
19565
|
+
case "unknown": return fail(`Not logged in to ${selection.syncServer}. Logged in to: ${selection.known.join(", ") || "(none)"}. Run: ${CMD} login --sync-server ${selection.syncServer}`);
|
|
19566
|
+
case "ambiguous": return fail(`Logged in to more than one Sync Server here: ${selection.known.join(", ")}. Say which with --sync-server <url>.`);
|
|
18139
19567
|
}
|
|
18140
19568
|
}
|
|
18141
19569
|
async function login(args) {
|
|
18142
19570
|
const path = defaultConfigPath();
|
|
18143
19571
|
const config = await readConfig(path) ?? emptyConfig();
|
|
18144
19572
|
const known = Object.keys(config.servers);
|
|
18145
|
-
const syncServer = normaliseSyncServer(args["sync-server"] ?? (known.length === 1 ? known[0] : await ask("Sync Server URL: ")));
|
|
19573
|
+
const syncServer = normaliseSyncServer(args["sync-server"] ?? (known.length === 1 ? known[0] : await ask("Sync Server URL: ", { hint: "pass --sync-server <url>" })));
|
|
18146
19574
|
if (!/^https?:\/\//.test(syncServer)) fail("The Sync Server must be an http(s) URL.");
|
|
18147
|
-
const pat = args.pat ?? process.env.ETHERPK_PAT ?? await ask(`Personal Access Token (account-wide, from ${syncServer}/account/tokens): `, {
|
|
19575
|
+
const pat = args.pat ?? process.env.ETHERPK_PAT ?? await ask(`Personal Access Token (account-wide, from ${syncServer}/account/tokens): `, {
|
|
19576
|
+
secret: true,
|
|
19577
|
+
hint: "set ETHERPK_PAT or pass --pat <token>"
|
|
19578
|
+
});
|
|
18148
19579
|
if (!pat) fail("A Personal Access Token is required.");
|
|
18149
19580
|
const account = await connectAccount({
|
|
18150
19581
|
syncServer,
|
|
18151
19582
|
pat
|
|
18152
19583
|
});
|
|
18153
|
-
console.log(`
|
|
18154
|
-
const byRecoveryCode = async () => unlockByRecoveryCode(account.api, process.env.ETHERPK_RECOVERY_CODE ?? await ask("Recovery Code: ", {
|
|
19584
|
+
console.log(`Connected to ${syncServer} as ${account.principal.email ?? account.principal.name ?? account.principal.id}.`);
|
|
19585
|
+
const byRecoveryCode = async () => unlockByRecoveryCode(account.api, process.env.ETHERPK_RECOVERY_CODE ?? await ask("Recovery Code: ", {
|
|
19586
|
+
secret: true,
|
|
19587
|
+
hint: "set ETHERPK_RECOVERY_CODE"
|
|
19588
|
+
}));
|
|
18155
19589
|
const vaultKey = args["recovery-code"] ? await byRecoveryCode() : await approveOrFallBack(account, byRecoveryCode);
|
|
18156
19590
|
config.servers[syncServer] = {
|
|
18157
19591
|
pat,
|
|
@@ -18160,7 +19594,7 @@ async function login(args) {
|
|
|
18160
19594
|
await writeConfig(path, config);
|
|
18161
19595
|
console.log(`Keys unlocked and cached in ${path} (owner-only). Anyone who can read your files on this machine can read this account, as with a signed-in browser.`);
|
|
18162
19596
|
const others = Object.keys(config.servers).filter((server) => server !== syncServer);
|
|
18163
|
-
if (others.length > 0) console.log(`Also
|
|
19597
|
+
if (others.length > 0) console.log(`Also logged in to ${others.join(", ")}; commands now need --sync-server <url> to say which.`);
|
|
18164
19598
|
await listGraphs({
|
|
18165
19599
|
syncServer,
|
|
18166
19600
|
pat,
|
|
@@ -18174,23 +19608,23 @@ async function login(args) {
|
|
|
18174
19608
|
* code instead; without a terminal the wait runs to its outcome.
|
|
18175
19609
|
*/
|
|
18176
19610
|
async function approveOrFallBack(account, byRecoveryCode) {
|
|
18177
|
-
const
|
|
19611
|
+
const controls = approvalWaitControls();
|
|
18178
19612
|
const io = {
|
|
18179
19613
|
say: (line) => console.log(line),
|
|
18180
|
-
sleep: (ms) =>
|
|
18181
|
-
signal:
|
|
19614
|
+
sleep: (ms) => sleepUnlessAborted(ms, controls.signal),
|
|
19615
|
+
signal: controls.signal,
|
|
18182
19616
|
clientUrl: account.clientUrl
|
|
18183
19617
|
};
|
|
18184
19618
|
const stdin = process.stdin;
|
|
18185
19619
|
const interactive = stdin.isTTY === true;
|
|
18186
|
-
const onKey = (chunk) =>
|
|
18187
|
-
|
|
18188
|
-
|
|
18189
|
-
|
|
18190
|
-
|
|
18191
|
-
|
|
18192
|
-
|
|
18193
|
-
|
|
19620
|
+
const onKey = (chunk) => controls.onKey(chunk.toString("utf8"));
|
|
19621
|
+
const quitSignals = [
|
|
19622
|
+
"SIGINT",
|
|
19623
|
+
"SIGTERM",
|
|
19624
|
+
"SIGHUP"
|
|
19625
|
+
];
|
|
19626
|
+
const onSignal = (name) => controls.onSignal(name);
|
|
19627
|
+
for (const name of quitSignals) process.once(name, onSignal);
|
|
18194
19628
|
if (interactive) {
|
|
18195
19629
|
console.log("(Press r to type your Recovery Code instead.)");
|
|
18196
19630
|
stdin.setRawMode(true);
|
|
@@ -18204,12 +19638,17 @@ async function approveOrFallBack(account, byRecoveryCode) {
|
|
|
18204
19638
|
if (!(error instanceof ApprovalAbandoned)) throw error;
|
|
18205
19639
|
abandoned = true;
|
|
18206
19640
|
} finally {
|
|
19641
|
+
for (const name of quitSignals) process.off(name, onSignal);
|
|
18207
19642
|
if (interactive) {
|
|
18208
19643
|
stdin.off("data", onKey);
|
|
18209
19644
|
stdin.setRawMode(false);
|
|
18210
19645
|
stdin.pause();
|
|
18211
19646
|
}
|
|
18212
19647
|
}
|
|
19648
|
+
if (controls.exitCode !== null) {
|
|
19649
|
+
console.log("");
|
|
19650
|
+
process.exit(controls.exitCode);
|
|
19651
|
+
}
|
|
18213
19652
|
if (!abandoned) throw new Error("unreachable");
|
|
18214
19653
|
console.log("Approval cancelled; unlocking with your Recovery Code instead.");
|
|
18215
19654
|
return byRecoveryCode();
|
|
@@ -18326,7 +19765,13 @@ async function publishCommand(args) {
|
|
|
18326
19765
|
"no-semantic": true
|
|
18327
19766
|
};
|
|
18328
19767
|
const { graph, graphName } = folder ? await openFolderForServe(folder, quiet) : await openSyncedForServe(wanted, quiet);
|
|
19768
|
+
const host = {
|
|
19769
|
+
env: process.env,
|
|
19770
|
+
cmd: CMD,
|
|
19771
|
+
via: "cli"
|
|
19772
|
+
};
|
|
18329
19773
|
try {
|
|
19774
|
+
await findPublication(graph, publication, host);
|
|
18330
19775
|
const configPath = defaultPublishFoldersPath(process.env);
|
|
18331
19776
|
const key = publishGraphKey(graph.backend, graph.graphId);
|
|
18332
19777
|
if (args.out?.trim()) {
|
|
@@ -18334,11 +19779,8 @@ async function publishCommand(args) {
|
|
|
18334
19779
|
await writePublishFolders(configPath, withPublishFolder(await readPublishFolders(configPath), key, publication, out));
|
|
18335
19780
|
console.error(`etherpk-mcp: publish folder for "${publication}" of "${graphName}" set to ${out} (remembered in ${configPath}).`);
|
|
18336
19781
|
} else if (!publishFolderOf(await readPublishFolders(configPath), key, publication)) fail(`No publish folder is set for "${publication}" of "${graphName}" on this machine. Pass --out <dir> once; it is remembered.`);
|
|
18337
|
-
const result = await publish(graph, { id: publication },
|
|
18338
|
-
|
|
18339
|
-
cmd: CMD
|
|
18340
|
-
});
|
|
18341
|
-
console.log(JSON.stringify(result, null, 2));
|
|
19782
|
+
const result = await publish(graph, { id: publication }, host);
|
|
19783
|
+
console.log(JSON.stringify(cliPublishOutput(result), null, 2));
|
|
18342
19784
|
if (!result.ok) process.exitCode = 1;
|
|
18343
19785
|
} catch (error) {
|
|
18344
19786
|
if (error instanceof ToolError) fail(`etherpk-mcp: ${error.message}`);
|
|
@@ -18420,6 +19862,11 @@ async function openSyncedForServe(wanted, args) {
|
|
|
18420
19862
|
const account = await connectAccount(login);
|
|
18421
19863
|
const vault = await openAccountVault(account.api, fromBase64Url(login.vaultKey));
|
|
18422
19864
|
const graphs = await account.api.listGraphs();
|
|
19865
|
+
const swept = await removeUnlistedGraphCaches(process.env, account.serverBaseUrl, account.principal.id, graphs.map((graph) => graph.id)).catch((error) => {
|
|
19866
|
+
console.error(`etherpk-mcp: could not tidy the cache of graphs this server no longer lists: ${error instanceof Error ? error.message : String(error)}`);
|
|
19867
|
+
return [];
|
|
19868
|
+
});
|
|
19869
|
+
if (swept.length > 0) console.error(`etherpk-mcp: removed this computer's copy of ${swept.length === 1 ? "a graph" : `${swept.length} graphs`} ${account.serverBaseUrl} no longer lists for you: ${swept.join(", ")}.`);
|
|
18423
19870
|
let graphId = graphs.find((graph) => graph.id === wanted)?.id;
|
|
18424
19871
|
let graphName = null;
|
|
18425
19872
|
if (!graphId) {
|
|
@@ -18431,6 +19878,8 @@ async function openSyncedForServe(wanted, args) {
|
|
|
18431
19878
|
}
|
|
18432
19879
|
if (!graphId) fail(`No synced graph on ${login.syncServer} is named or identified by "${wanted}". Run: ${CMD} graphs`);
|
|
18433
19880
|
const { record, keyring } = resolveGraphById(graphs, vault, graphId);
|
|
19881
|
+
const persistDir = graphCacheDir(process.env, account.serverBaseUrl, graphId);
|
|
19882
|
+
await stampGraphCacheOwner(persistDir, account.principal.id);
|
|
18434
19883
|
console.error(`etherpk-mcp: opening graph ${graphId} on ${account.serverBaseUrl}…`);
|
|
18435
19884
|
const graph = await openHeadlessGraph({
|
|
18436
19885
|
graphId,
|
|
@@ -18440,11 +19889,12 @@ async function openSyncedForServe(wanted, args) {
|
|
|
18440
19889
|
token: account.tokenFor(graphId),
|
|
18441
19890
|
presenceName: `Agent on ${hostname()}`,
|
|
18442
19891
|
readyTimeoutMs: 2e4,
|
|
18443
|
-
persistDir
|
|
19892
|
+
persistDir,
|
|
18444
19893
|
assets: { baseUrl: account.serverBaseUrl },
|
|
18445
19894
|
embeddingModel: embeddingModelFor(args),
|
|
18446
19895
|
onSemanticProgress: reportSemanticProgress,
|
|
18447
19896
|
onError: (error) => console.error(`etherpk-mcp: ${error.message}`),
|
|
19897
|
+
onAccessLost: (loss) => console.error(loss.kind === "membership" ? `etherpk-mcp: ${account.serverBaseUrl} ended this account's access to graph ${graphId}: it left, was removed, or the graph was deleted. Stopped syncing. Run: ${CMD} graphs` : `etherpk-mcp: the token for ${account.serverBaseUrl} was revoked or is no longer valid. Stopped syncing. Run: ${CMD} login --sync-server ${login.syncServer}`),
|
|
18448
19898
|
publishName: createGraphNamePublisher({
|
|
18449
19899
|
api: account.api,
|
|
18450
19900
|
keyring,
|