@drupflare/spool 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +444 -18
- package/dist/cdn/client.js +1 -1
- package/dist/cdn/manifest.json +7 -7
- package/dist/cdn/spool-sw.js +1 -1
- package/dist/client/activity.d.ts +37 -0
- package/dist/client/activity.js +114 -0
- package/dist/client/entry.js +2 -1
- package/dist/client/index.d.ts +1 -0
- package/dist/client/index.js +1 -0
- package/dist/media/embeds.d.ts +60 -0
- package/dist/media/embeds.js +135 -0
- package/dist/media/headers.d.ts +13 -0
- package/dist/media/headers.js +44 -0
- package/dist/media/html.d.ts +27 -0
- package/dist/media/html.js +93 -0
- package/dist/media/image.d.ts +222 -0
- package/dist/media/image.js +413 -0
- package/dist/media/index.d.ts +6 -0
- package/dist/media/index.js +6 -0
- package/dist/media/lazy.d.ts +26 -0
- package/dist/media/lazy.js +74 -0
- package/dist/media/responsive.d.ts +45 -0
- package/dist/media/responsive.js +98 -0
- package/dist/media/video.d.ts +30 -0
- package/dist/media/video.js +75 -0
- package/dist/protocol/envelope.d.ts +2 -0
- package/dist/protocol/envelope.js +12 -2
- package/dist/protocol/errors.d.ts +1 -1
- package/dist/protocol/hash.d.ts +2 -0
- package/dist/protocol/hash.js +8 -0
- package/dist/protocol/index.d.ts +4 -0
- package/dist/protocol/index.js +4 -0
- package/dist/protocol/inline.d.ts +14 -0
- package/dist/protocol/inline.js +43 -0
- package/dist/protocol/navigation.d.ts +81 -0
- package/dist/protocol/navigation.js +249 -0
- package/dist/protocol/normalise.d.ts +4 -3
- package/dist/protocol/normalise.js +6 -4
- package/dist/protocol/provision.d.ts +6 -1
- package/dist/protocol/provision.js +15 -5
- package/dist/protocol/query.d.ts +80 -0
- package/dist/protocol/query.js +153 -0
- package/dist/protocol/route.d.ts +80 -4
- package/dist/protocol/route.js +63 -0
- package/dist/proxy/bases.d.ts +53 -0
- package/dist/proxy/bases.js +76 -0
- package/dist/proxy/dictionary.d.ts +58 -0
- package/dist/proxy/dictionary.js +197 -0
- package/dist/proxy/index.d.ts +2 -0
- package/dist/proxy/index.js +2 -0
- package/dist/proxy/proxy.d.ts +47 -0
- package/dist/proxy/proxy.js +140 -15
- package/dist/report/capacity.d.ts +59 -0
- package/dist/report/capacity.js +76 -0
- package/dist/report/channels.d.ts +60 -0
- package/dist/report/channels.js +65 -0
- package/dist/report/derive.d.ts +2 -0
- package/dist/report/derive.js +6 -2
- package/dist/report/index.d.ts +4 -0
- package/dist/report/index.js +4 -0
- package/dist/report/institution.d.ts +97 -0
- package/dist/report/institution.js +79 -0
- package/dist/report/ledger.d.ts +197 -0
- package/dist/report/ledger.js +308 -0
- package/dist/report/measurement.d.ts +1 -1
- package/dist/report/measurement.js +7 -1
- package/dist/report/traffic.d.ts +11 -2
- package/dist/report/traffic.js +13 -3
- package/dist/server/embedded.d.ts +3 -3
- package/dist/server/embedded.js +3 -3
- package/dist/server/index.d.ts +1 -0
- package/dist/server/index.js +1 -0
- package/dist/server/spool.d.ts +20 -0
- package/dist/server/spool.js +62 -9
- package/dist/sw/render.d.ts +3 -1
- package/dist/sw/render.js +14 -1
- package/dist/sw/replica.d.ts +2 -0
- package/dist/sw/replica.js +6 -0
- package/dist/sw/router.js +2 -2
- package/dist/sw/runtime.js +104 -9
- package/dist/trim/tokens.d.ts +4 -0
- package/dist/trim/tokens.js +1 -1
- package/package.json +10 -4
package/dist/sw/replica.js
CHANGED
|
@@ -18,6 +18,7 @@ export class Replica {
|
|
|
18
18
|
#tags = new Map();
|
|
19
19
|
#cursor = null;
|
|
20
20
|
#syncedAt = 0;
|
|
21
|
+
#epoch = 0;
|
|
21
22
|
#site;
|
|
22
23
|
#policies = new Map();
|
|
23
24
|
/** records and projections; untouched until a page that references them is stored or shown */
|
|
@@ -90,6 +91,10 @@ export class Replica {
|
|
|
90
91
|
async ids() {
|
|
91
92
|
return (await this.#all()).map((s) => s.artifact.id);
|
|
92
93
|
}
|
|
94
|
+
/** counts the wipes so far; a write that began before one and ends after it can tell and drop itself */
|
|
95
|
+
get epoch() {
|
|
96
|
+
return this.#epoch;
|
|
97
|
+
}
|
|
93
98
|
/** whether no dependency of `artifact` has changed since it was built */
|
|
94
99
|
isCurrent(artifact) {
|
|
95
100
|
return isCurrent(artifact, (tag) => this.#tags.get(tag) ?? 0);
|
|
@@ -208,6 +213,7 @@ export class Replica {
|
|
|
208
213
|
}
|
|
209
214
|
/** empties the replica; `cursor` is where it restarts */
|
|
210
215
|
async wipe(cursor = null) {
|
|
216
|
+
this.#epoch += 1;
|
|
211
217
|
this.#memory.clear();
|
|
212
218
|
await this.#storage.clearArtifacts();
|
|
213
219
|
await this.graph.clear();
|
package/dist/sw/router.js
CHANGED
|
@@ -31,8 +31,8 @@ export function classify(config, request, origin) {
|
|
|
31
31
|
if (request.method !== 'GET')
|
|
32
32
|
return { kind: 'pass' };
|
|
33
33
|
const hit = matchRoute(config.routes, url.pathname);
|
|
34
|
-
// an artifact is keyed by path alone, so a
|
|
35
|
-
const queried = url.search !== '';
|
|
34
|
+
// an artifact is keyed by path alone, so only a route with a query schema answers a query
|
|
35
|
+
const queried = url.search !== '' && !hit?.route.query;
|
|
36
36
|
if (request.mode !== 'navigate') {
|
|
37
37
|
if (hit?.route.strategy === 'static' && !request.headers.has('range')) {
|
|
38
38
|
return { kind: 'static', route: hit.route, path: url.pathname + url.search };
|
package/dist/sw/runtime.js
CHANGED
|
@@ -3,7 +3,10 @@ import { MAX_WANT } from '../protocol/beacon.js';
|
|
|
3
3
|
import { applyStamp } from '../protocol/collector.js';
|
|
4
4
|
import { PATHS } from '../protocol/config.js';
|
|
5
5
|
import { decodeEnvelope, decodeEnvelopeOrClear, HEADERS } from '../protocol/envelope.js';
|
|
6
|
+
import { shellHash } from '../protocol/hash.js';
|
|
7
|
+
import { readInline } from '../protocol/inline.js';
|
|
6
8
|
import { manifestRoutes, parseManifest } from '../protocol/manifest.js';
|
|
9
|
+
import { PROOF_MISMATCH } from '../protocol/query.js';
|
|
7
10
|
import { matchRoute } from '../protocol/route.js';
|
|
8
11
|
import { encodeHeld, parseSyncResponse } from '../protocol/sync.js';
|
|
9
12
|
import { engineOf, FROM_EXTERNAL, loadsToBeacon, REASON_TIER, viewToBeacon } from '../protocol/view.js';
|
|
@@ -483,10 +486,13 @@ export function createRuntime(deps) {
|
|
|
483
486
|
await early(envelope, navigation, after?.signedOut);
|
|
484
487
|
// a cacheable GET may be answered by the HTTP cache with no radio wake, so only these count
|
|
485
488
|
const moment = navigation || request.method !== 'GET';
|
|
486
|
-
const
|
|
489
|
+
const clone = after?.clone;
|
|
490
|
+
const copy = (typeof clone === 'function' ? clone(response, envelope) : clone)
|
|
491
|
+
? response.clone()
|
|
492
|
+
: null;
|
|
487
493
|
track((async () => {
|
|
488
494
|
try {
|
|
489
|
-
await after?.run(copy);
|
|
495
|
+
await after?.run(copy, envelope);
|
|
490
496
|
}
|
|
491
497
|
catch {
|
|
492
498
|
// a view that cannot be recorded must not stop the response being observed
|
|
@@ -495,7 +501,8 @@ export function createRuntime(deps) {
|
|
|
495
501
|
})());
|
|
496
502
|
return response;
|
|
497
503
|
}
|
|
498
|
-
|
|
504
|
+
/** `search` is the request's query string; the stored artifact is always the path's own */
|
|
505
|
+
async function decide(path, route, dry, search = '') {
|
|
499
506
|
let side = publicSide;
|
|
500
507
|
if (route.scope === 'principal') {
|
|
501
508
|
if (!host)
|
|
@@ -581,9 +588,13 @@ export function createRuntime(deps) {
|
|
|
581
588
|
await rep.drop([path]);
|
|
582
589
|
return { reason: 'refused', staleSec: age };
|
|
583
590
|
}
|
|
584
|
-
const rendered = await renderLocal(config, route, plain.artifact, renderers);
|
|
585
|
-
if (!rendered.ok)
|
|
591
|
+
const rendered = await renderLocal(config, route, plain.artifact, renderers, search);
|
|
592
|
+
if (!rendered.ok) {
|
|
593
|
+
// a collection that fails its proof is dropped, so the next network view fetches it whole
|
|
594
|
+
if (rendered.reason === PROOF_MISMATCH)
|
|
595
|
+
await rep.drop([path]);
|
|
586
596
|
return { reason: 'refused', staleSec: age };
|
|
597
|
+
}
|
|
587
598
|
if (fromCatalogue) {
|
|
588
599
|
await rep.open(path);
|
|
589
600
|
if (REASON_TIER[reason] === 'local')
|
|
@@ -612,7 +623,7 @@ export function createRuntime(deps) {
|
|
|
612
623
|
const gapMs = lastViewAt === null ? null : at - lastViewAt;
|
|
613
624
|
lastViewAt = at;
|
|
614
625
|
currentPath = route.catalogue ? path : null;
|
|
615
|
-
const decision = await decide(path, route, shadow);
|
|
626
|
+
const decision = await decide(path, route, shadow, new URL(request.url).search);
|
|
616
627
|
const from = referrerRoute(request.referrer);
|
|
617
628
|
const id = crypto.randomUUID();
|
|
618
629
|
latestView.delete(path);
|
|
@@ -656,11 +667,13 @@ export function createRuntime(deps) {
|
|
|
656
667
|
if (decision.html === undefined) {
|
|
657
668
|
const keeps = route.strategy === 'static';
|
|
658
669
|
const response = await passthrough(request, true, {
|
|
659
|
-
clone: keeps,
|
|
670
|
+
clone: keeps || ((res, envelope) => derivable(path, route, res, envelope)),
|
|
660
671
|
route,
|
|
661
|
-
run: async (copy) => {
|
|
672
|
+
run: async (copy, envelope) => {
|
|
662
673
|
if (keeps)
|
|
663
674
|
await capture(path, route, copy);
|
|
675
|
+
else
|
|
676
|
+
await derive(path, route, copy, envelope);
|
|
664
677
|
await record(null);
|
|
665
678
|
}
|
|
666
679
|
});
|
|
@@ -688,7 +701,7 @@ export function createRuntime(deps) {
|
|
|
688
701
|
}
|
|
689
702
|
/** answers a GET for a `data` route from the stored value; the page makes no request */
|
|
690
703
|
async function dataFile(request, path, route) {
|
|
691
|
-
const decision = await decide(path, route, false);
|
|
704
|
+
const decision = await decide(path, route, false, new URL(request.url).search);
|
|
692
705
|
if (decision.html !== undefined) {
|
|
693
706
|
ledger.serve(path, decision.reason === 'catalogue');
|
|
694
707
|
const headers = new Headers(decision.headers);
|
|
@@ -771,6 +784,88 @@ export function createRuntime(deps) {
|
|
|
771
784
|
ledger.spend(path, { bytes: bytes.byteLength, requests: 0 });
|
|
772
785
|
}
|
|
773
786
|
}
|
|
787
|
+
/** whether a cold navigation's body can stand in for the artifact its response advertises */
|
|
788
|
+
function derivable(path, route, response, envelope) {
|
|
789
|
+
const advertised = envelope?.artifact;
|
|
790
|
+
return ((route.strategy === 'assembly' || route.strategy === 'template') &&
|
|
791
|
+
route.scope === 'public' &&
|
|
792
|
+
envelope?.protocol === PROTOCOL_VERSION &&
|
|
793
|
+
advertised?.hash !== undefined &&
|
|
794
|
+
advertised.id === path &&
|
|
795
|
+
advertised.scope === 'public' &&
|
|
796
|
+
modeFor(config, route) !== 'shadow' &&
|
|
797
|
+
!declined.has(path) &&
|
|
798
|
+
response.status === 200 &&
|
|
799
|
+
!response.redirected &&
|
|
800
|
+
/text\/html/i.test(response.headers.get('content-type') ?? ''));
|
|
801
|
+
}
|
|
802
|
+
/**
|
|
803
|
+
* The artifact a navigation body carries, or null. An `assembly` page is the artifact's shell
|
|
804
|
+
* once the client tag is out, and a `template` page holds the artifact's JSON in its inline
|
|
805
|
+
* element. Either must hash to the value `Spool-Artifact` named.
|
|
806
|
+
*/
|
|
807
|
+
async function carried(path, route, text, advertised) {
|
|
808
|
+
if (route.strategy === 'template') {
|
|
809
|
+
const json = readInline(text);
|
|
810
|
+
if (json === null || (await shellHash(json)) !== advertised.hash)
|
|
811
|
+
return null;
|
|
812
|
+
const artifact = parseArtifact(JSON.parse(json));
|
|
813
|
+
if (artifact.id !== path || artifact.scope !== 'public')
|
|
814
|
+
return null;
|
|
815
|
+
if (artifact.strategy !== 'template')
|
|
816
|
+
return null;
|
|
817
|
+
return { artifact, bytes: encoder.encode(json).length };
|
|
818
|
+
}
|
|
819
|
+
const at = text.indexOf(config.clientTag);
|
|
820
|
+
const shells = at < 0 ? [text] : [text.slice(0, at) + text.slice(at + config.clientTag.length), text];
|
|
821
|
+
for (const shell of shells) {
|
|
822
|
+
if ((await shellHash(shell)) !== advertised.hash)
|
|
823
|
+
continue;
|
|
824
|
+
return {
|
|
825
|
+
artifact: {
|
|
826
|
+
protocol: PROTOCOL_VERSION,
|
|
827
|
+
id: path,
|
|
828
|
+
scope: 'public',
|
|
829
|
+
strategy: 'assembly',
|
|
830
|
+
deps: [...new Set(advertised.deps)],
|
|
831
|
+
vector: advertised.version,
|
|
832
|
+
body: { shell }
|
|
833
|
+
},
|
|
834
|
+
bytes: 0
|
|
835
|
+
};
|
|
836
|
+
}
|
|
837
|
+
return null;
|
|
838
|
+
}
|
|
839
|
+
/**
|
|
840
|
+
* Keeps the artifact a navigation's own body carries, when it matches the hash `Spool-Artifact`
|
|
841
|
+
* named. Anything else leaves nothing stored, and the advertised artifact is then fetched as
|
|
842
|
+
* before.
|
|
843
|
+
*/
|
|
844
|
+
async function derive(path, route, copy, envelope) {
|
|
845
|
+
const advertised = envelope?.artifact;
|
|
846
|
+
if (!copy || !advertised?.hash || !derivable(path, route, copy, envelope))
|
|
847
|
+
return;
|
|
848
|
+
// a wipe while the body is read means the copy predates it, so it is not kept
|
|
849
|
+
const epoch = replica.epoch;
|
|
850
|
+
try {
|
|
851
|
+
const bytes = await readCapped(copy, MAX_STATIC_BYTES);
|
|
852
|
+
if (!bytes)
|
|
853
|
+
return;
|
|
854
|
+
const text = new TextDecoder('utf-8', { fatal: true }).decode(bytes);
|
|
855
|
+
const found = await carried(path, route, text, {
|
|
856
|
+
...advertised,
|
|
857
|
+
hash: advertised.hash
|
|
858
|
+
});
|
|
859
|
+
if (!found || (await replica.get(path)) || replica.epoch !== epoch)
|
|
860
|
+
return;
|
|
861
|
+
if (await replica.put(found.artifact, route.persist)) {
|
|
862
|
+
ledger.spend(path, { bytes: found.bytes, requests: 0 });
|
|
863
|
+
}
|
|
864
|
+
}
|
|
865
|
+
catch {
|
|
866
|
+
// an unreadable body leaves the network fetch as the only way to the artifact
|
|
867
|
+
}
|
|
868
|
+
}
|
|
774
869
|
async function staticFile(request, path, route) {
|
|
775
870
|
const decision = await decide(path, route, false);
|
|
776
871
|
if (decision.response) {
|
package/dist/trim/tokens.d.ts
CHANGED
|
@@ -13,6 +13,10 @@ export type Tag = {
|
|
|
13
13
|
name: string;
|
|
14
14
|
attributes: Attribute[];
|
|
15
15
|
body?: string;
|
|
16
|
+
/** where the `<` sits in the HTML */
|
|
17
|
+
start: number;
|
|
18
|
+
/** where the closing `>` sits, or the length of the HTML when the tag never closes */
|
|
19
|
+
end: number;
|
|
16
20
|
};
|
|
17
21
|
/** the value of the first attribute called `name` on `tag` */
|
|
18
22
|
export declare const attr: (tag: Tag, name: string) => string | undefined;
|
package/dist/trim/tokens.js
CHANGED
|
@@ -76,7 +76,7 @@ export function tagsOf(html) {
|
|
|
76
76
|
}
|
|
77
77
|
attributes.push({ name, value, start, end: i });
|
|
78
78
|
}
|
|
79
|
-
const tag = { name: found[0].toLowerCase(), attributes };
|
|
79
|
+
const tag = { name: found[0].toLowerCase(), attributes, start: open, end: i };
|
|
80
80
|
at = Math.min(html.length, i + 1);
|
|
81
81
|
if (tag.name === 'script' || tag.name === 'style') {
|
|
82
82
|
const end = new RegExp(`</${tag.name}\\s*>`, 'gi');
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@drupflare/spool",
|
|
3
3
|
"description": "Client-Side Page Rendering and Execution Framework",
|
|
4
|
-
"version": "0.
|
|
4
|
+
"version": "0.3.0",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|
|
7
7
|
"types": "./dist/index.d.ts",
|
|
@@ -46,6 +46,10 @@
|
|
|
46
46
|
"types": "./dist/report/index.d.ts",
|
|
47
47
|
"default": "./dist/report/index.js"
|
|
48
48
|
},
|
|
49
|
+
"./media": {
|
|
50
|
+
"types": "./dist/media/index.d.ts",
|
|
51
|
+
"default": "./dist/media/index.js"
|
|
52
|
+
},
|
|
49
53
|
"./trim": {
|
|
50
54
|
"types": "./dist/trim/index.d.ts",
|
|
51
55
|
"default": "./dist/trim/index.js"
|
|
@@ -83,11 +87,13 @@
|
|
|
83
87
|
"measure:iphone": "bun scripts/measure/iphone.ts",
|
|
84
88
|
"measure:traces": "bun scripts/measure/traces.ts",
|
|
85
89
|
"measure:payload": "bun scripts/measure/payload.ts",
|
|
86
|
-
"measure:trim": "bun scripts/measure/trim.ts"
|
|
90
|
+
"measure:trim": "bun scripts/measure/trim.ts",
|
|
91
|
+
"measure:nav-replay": "bun scripts/measure/nav-replay.ts",
|
|
92
|
+
"measure:embed-markup": "bun scripts/measure/embed-markup.ts"
|
|
87
93
|
},
|
|
88
94
|
"devDependencies": {
|
|
89
95
|
"@azure/functions": "^4.16.5",
|
|
90
|
-
"@cloudflare/vitest-pool-workers": "^0.
|
|
96
|
+
"@cloudflare/vitest-pool-workers": "^0.23.0",
|
|
91
97
|
"@cloudflare/workers-types": "^5.20260811.1",
|
|
92
98
|
"@google-cloud/functions-framework": "^5.0.5",
|
|
93
99
|
"@hapi/hapi": "^21.4.10",
|
|
@@ -100,7 +106,7 @@
|
|
|
100
106
|
"@types/bun": "^1.4.3",
|
|
101
107
|
"@types/express": "^5.0.6",
|
|
102
108
|
"@types/koa": "^3.0.3",
|
|
103
|
-
"@types/node": "^
|
|
109
|
+
"@types/node": "^26.0.0",
|
|
104
110
|
"@vitest/coverage-istanbul": "^4.1.0",
|
|
105
111
|
"elysia": "^1.4.30",
|
|
106
112
|
"express": "^5.3.0",
|