@instafy/cli 0.2.3 → 0.2.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -0
- package/dist/bind/LICENSE +661 -0
- package/dist/bind/bin/instafy-bind.mjs +542 -0
- package/dist/bind/client/bind-client.js +1219 -0
- package/dist/bind/src/apply.mjs +293 -0
- package/dist/bind/src/bound.mjs +202 -0
- package/dist/bind/src/budget.mjs +58 -0
- package/dist/bind/src/check.mjs +176 -0
- package/dist/bind/src/config.mjs +180 -0
- package/dist/bind/src/controller.mjs +244 -0
- package/dist/bind/src/executors/agent.mjs +336 -0
- package/dist/bind/src/executors/script.mjs +163 -0
- package/dist/bind/src/handler-kit.mjs +150 -0
- package/dist/bind/src/init.mjs +238 -0
- package/dist/bind/src/jobs.mjs +316 -0
- package/dist/bind/src/net.mjs +281 -0
- package/dist/bind/src/page-fetch.mjs +43 -0
- package/dist/bind/src/preview-host.mjs +114 -0
- package/dist/bind/src/protocol.mjs +103 -0
- package/dist/bind/src/records-schema.mjs +580 -0
- package/dist/bind/src/records.mjs +161 -0
- package/dist/bind/src/save-owner.mjs +492 -0
- package/dist/bind/src/schema-lite.mjs +212 -0
- package/dist/bind/src/server.mjs +908 -0
- package/dist/bind/src/sse.mjs +119 -0
- package/dist/bind/src/starters/blank/app.js +17 -0
- package/dist/bind/src/starters/blank/index.html +22 -0
- package/dist/bind/src/starters/checklist/app.js +171 -0
- package/dist/bind/src/starters/checklist/bound.json +29 -0
- package/dist/bind/src/starters/checklist/data.json +26 -0
- package/dist/bind/src/starters/checklist/index.html +23 -0
- package/dist/bind/src/starters/compare/app.js +316 -0
- package/dist/bind/src/starters/compare/bound.json +26 -0
- package/dist/bind/src/starters/compare/data.json +170 -0
- package/dist/bind/src/starters/compare/index.html +23 -0
- package/dist/bind/src/starters/handlers/read_offer.mjs +55 -0
- package/dist/bind/src/starters/handlers/set_check.mjs +41 -0
- package/dist/bind/src/starters/ranked/app.js +166 -0
- package/dist/bind/src/starters/ranked/bound.json +26 -0
- package/dist/bind/src/starters/ranked/data.json +142 -0
- package/dist/bind/src/starters/ranked/index.html +23 -0
- package/dist/bind/src/starters/site.js +888 -0
- package/dist/bind/src/starters/theme.css +492 -0
- package/dist/bind/src/starters/timeline/app.js +143 -0
- package/dist/bind/src/starters/timeline/data.json +41 -0
- package/dist/bind/src/starters/timeline/index.html +22 -0
- package/dist/bind/src/static.mjs +141 -0
- package/dist/bind/src/tokens.mjs +102 -0
- package/dist/bind/src/try.mjs +139 -0
- package/dist/bind/src/web.mjs +191 -0
- package/dist/cli.js +2117 -365
- package/package.json +3 -2
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
// instafy.records@1 records: normalise, validate, merge.
|
|
2
|
+
// The bind client carries the same merge rules in client/bind-client.js.
|
|
3
|
+
import { RECORDS_SCHEMA } from "./records-schema.mjs";
|
|
4
|
+
import { validate } from "./schema-lite.mjs";
|
|
5
|
+
|
|
6
|
+
export { RECORDS_SCHEMA };
|
|
7
|
+
export const RECORDS_SCHEMA_ID = "instafy.records@1";
|
|
8
|
+
export const RECORD_TYPES = ["claim", "offer", "item"];
|
|
9
|
+
export const LANES = ["link", "official", "open", "capture", "feed", "assist", "local"];
|
|
10
|
+
export const RECORDS_PATH_RE = /^data\/records\/[A-Za-z0-9_.-]{1,80}\.json$/;
|
|
11
|
+
export const MAX_RECORDS = 2000;
|
|
12
|
+
export const MAX_FILE_BYTES = 200 * 1024;
|
|
13
|
+
|
|
14
|
+
/** Copy with canonical names: `observer` becomes `observed_by`, `ttl` becomes `ttl_s`. */
|
|
15
|
+
export function normalizeRecord(record) {
|
|
16
|
+
if (!record || typeof record !== "object" || Array.isArray(record)) return record;
|
|
17
|
+
const out = structuredClone(record);
|
|
18
|
+
const prov = out.prov;
|
|
19
|
+
if (prov && typeof prov === "object" && !Array.isArray(prov)) {
|
|
20
|
+
if (prov.observer !== undefined && prov.observed_by === undefined) prov.observed_by = prov.observer;
|
|
21
|
+
delete prov.observer;
|
|
22
|
+
if (prov.ttl !== undefined && prov.ttl_s === undefined) prov.ttl_s = prov.ttl;
|
|
23
|
+
delete prov.ttl;
|
|
24
|
+
}
|
|
25
|
+
return out;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/** Validate one record against its type's definition. Returns [{path, message}]. */
|
|
29
|
+
export function validateRecord(record) {
|
|
30
|
+
if (!record || typeof record !== "object" || !RECORD_TYPES.includes(record.type)) {
|
|
31
|
+
return [{ path: "$.type", message: 'type must be "claim", "offer" or "item"' }];
|
|
32
|
+
}
|
|
33
|
+
return validate(RECORDS_SCHEMA.$defs[record.type], record, { root: RECORDS_SCHEMA, max: 8 });
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/** Validate a whole records file object. */
|
|
37
|
+
export function validateRecordsFile(obj) {
|
|
38
|
+
return validate(RECORDS_SCHEMA, obj, { max: 30 });
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export function recordKey(record) {
|
|
42
|
+
if (record.type === "claim") return `c:${record.cid}`;
|
|
43
|
+
if (record.type === "offer") return `o:${record.oid}`;
|
|
44
|
+
return `i:${record.iid}`;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
const observedAt = (record) => Date.parse(record?.prov?.observed_at || "") || 0;
|
|
48
|
+
|
|
49
|
+
/** Lanes whose records come from a model or a capture. They never replace a record from another lane. */
|
|
50
|
+
export const LOW_TRUST_LANES = ["assist", "capture"];
|
|
51
|
+
const laneOf = (record) => record?.prov?.lane ?? null;
|
|
52
|
+
|
|
53
|
+
/** May `next` replace `previous`? A newer observation wins, except that assist/capture never displace other lanes. */
|
|
54
|
+
export function canReplace(previous, next) {
|
|
55
|
+
if (LOW_TRUST_LANES.includes(laneOf(next)) && !LOW_TRUST_LANES.includes(laneOf(previous))) return false;
|
|
56
|
+
return observedAt(next) >= observedAt(previous);
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* Merge incoming records into existing ones.
|
|
61
|
+
* - Same key (cid/oid/iid): the newer prov.observed_at wins.
|
|
62
|
+
* - Claims with a different key but the same (ref, attr, prov.source, prov.lane): the newer replaces the older.
|
|
63
|
+
* - An assist or capture record never replaces a record from another lane.
|
|
64
|
+
* Returns {records, applied (the incoming records that were stored), added, replaced, kept, dropped}.
|
|
65
|
+
*/
|
|
66
|
+
export function mergeRecords(existing, incoming) {
|
|
67
|
+
const byKey = new Map();
|
|
68
|
+
for (const record of existing) byKey.set(recordKey(record), record);
|
|
69
|
+
const applied = [];
|
|
70
|
+
let added = 0;
|
|
71
|
+
let replaced = 0;
|
|
72
|
+
let kept = 0;
|
|
73
|
+
let dropped = 0;
|
|
74
|
+
for (const record of incoming) {
|
|
75
|
+
const key = recordKey(record);
|
|
76
|
+
const previous = byKey.get(key);
|
|
77
|
+
if (previous) {
|
|
78
|
+
if (canReplace(previous, record)) {
|
|
79
|
+
byKey.set(key, record);
|
|
80
|
+
applied.push(record);
|
|
81
|
+
replaced += 1;
|
|
82
|
+
} else {
|
|
83
|
+
kept += 1;
|
|
84
|
+
}
|
|
85
|
+
continue;
|
|
86
|
+
}
|
|
87
|
+
if (record.type === "claim") {
|
|
88
|
+
let older = null;
|
|
89
|
+
for (const [otherKey, other] of byKey) {
|
|
90
|
+
if (
|
|
91
|
+
other.type === "claim" && other.ref === record.ref && other.attr === record.attr &&
|
|
92
|
+
other.prov?.source === record.prov?.source && laneOf(other) === laneOf(record)
|
|
93
|
+
) {
|
|
94
|
+
older = [otherKey, other];
|
|
95
|
+
break;
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
if (older) {
|
|
99
|
+
if (canReplace(older[1], record)) {
|
|
100
|
+
byKey.delete(older[0]);
|
|
101
|
+
byKey.set(key, record);
|
|
102
|
+
applied.push(record);
|
|
103
|
+
replaced += 1;
|
|
104
|
+
dropped += 1;
|
|
105
|
+
} else {
|
|
106
|
+
kept += 1;
|
|
107
|
+
}
|
|
108
|
+
continue;
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
byKey.set(key, record);
|
|
112
|
+
applied.push(record);
|
|
113
|
+
added += 1;
|
|
114
|
+
}
|
|
115
|
+
let records = [...byKey.values()];
|
|
116
|
+
if (records.length > MAX_RECORDS) {
|
|
117
|
+
records = records.sort((a, b) => observedAt(b) - observedAt(a)).slice(0, MAX_RECORDS);
|
|
118
|
+
}
|
|
119
|
+
const stored = new Set(records);
|
|
120
|
+
return { records, applied: applied.filter((r) => stored.has(r)), added, replaced, kept, dropped };
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/** True when observed_at + ttl_s lies in the past. */
|
|
124
|
+
export function isExpired(record, now = Date.now()) {
|
|
125
|
+
const ttl = record?.prov?.ttl_s;
|
|
126
|
+
if (!Number.isFinite(ttl) || ttl <= 0) return false;
|
|
127
|
+
return observedAt(record) + ttl * 1000 < now;
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
/**
|
|
131
|
+
* Keep a records file under `limit` bytes: drop expired records first, then the oldest.
|
|
132
|
+
* Returns {records, trimmed}.
|
|
133
|
+
*/
|
|
134
|
+
export function trimRecords(records, limit = MAX_FILE_BYTES, now = Date.now()) {
|
|
135
|
+
const sizeOf = (r) => Buffer.byteLength(JSON.stringify(r)) + 1;
|
|
136
|
+
let total = records.reduce((sum, r) => sum + sizeOf(r), 0) + 200;
|
|
137
|
+
if (total <= limit) return { records, trimmed: 0 };
|
|
138
|
+
const order = records
|
|
139
|
+
.map((r) => ({ r, expired: isExpired(r, now), at: observedAt(r) }))
|
|
140
|
+
.sort((a, b) => Number(b.expired) - Number(a.expired) || a.at - b.at);
|
|
141
|
+
const drop = new Set();
|
|
142
|
+
for (const entry of order) {
|
|
143
|
+
if (total <= limit) break;
|
|
144
|
+
drop.add(entry.r);
|
|
145
|
+
total -= sizeOf(entry.r);
|
|
146
|
+
}
|
|
147
|
+
return { records: records.filter((r) => !drop.has(r)), trimmed: drop.size };
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
export function emptyRecordsFile() {
|
|
151
|
+
return { schema: RECORDS_SCHEMA_ID, writer: {}, attribution: [], records: [] };
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/** Stable record id: "<prefix>_<slug>", within the rid pattern. */
|
|
155
|
+
export function rid(prefix, ...parts) {
|
|
156
|
+
const slug = parts
|
|
157
|
+
.map((part) => String(part ?? "").normalize("NFKD").replace(/[^A-Za-z0-9_.:-]+/g, "_").replace(/^_+|_+$/g, ""))
|
|
158
|
+
.filter(Boolean)
|
|
159
|
+
.join("_");
|
|
160
|
+
return `${prefix}_${slug}`.slice(0, 96);
|
|
161
|
+
}
|
|
@@ -0,0 +1,492 @@
|
|
|
1
|
+
// The save owner of a site's own writes: one mechanism for every site backend. A backend that ships elsewhere carries
|
|
2
|
+
// this file byte for byte.
|
|
3
|
+
//
|
|
4
|
+
// A site backend (the binder, a map host) writes data files between turns, on behalf of a click: records and
|
|
5
|
+
// data/version.json. In an Instafy space nothing owned those writes. No turn reported them, so they sat as changed
|
|
6
|
+
// files in the space's checkout until a stop parked them as "unsaved work", and Studio told everyone that some work
|
|
7
|
+
// wasn't saved.
|
|
8
|
+
//
|
|
9
|
+
// The backend now owns them. In a folder of an Instafy space (a `.instafy/.git` checkout above the site), it commits
|
|
10
|
+
// the files it wrote to the checkout's branch, locally, a few seconds after the last write. That is the save path a
|
|
11
|
+
// turn's own commits take: the next pre-turn refresh, turn save or stop publishes the branch's commits to canonical.
|
|
12
|
+
// No credential and no network call are involved.
|
|
13
|
+
//
|
|
14
|
+
// The commit is made the way the space's origin changes the checkout, so the two never interleave:
|
|
15
|
+
// - Under the origin's own workspace lock (`.instafy/origin-apply.lock`, an exclusive flock). While the origin
|
|
16
|
+
// publishes, refreshes or flushes for a stop, the save waits and is tried again a few seconds later.
|
|
17
|
+
// - With git plumbing only: the bytes the backend wrote are hashed with no filters into a temporary index, the commit
|
|
18
|
+
// is built from that, and HEAD moves only if it is still the commit the save started from. Then the checkout's own
|
|
19
|
+
// index takes the same entries, so the files are no longer changes.
|
|
20
|
+
// - With no hook, no fsmonitor, no signing, no global or system git config and none of the backend's environment
|
|
21
|
+
// (credentials included): the checkout's config is the workspace's to write, so nothing in it runs.
|
|
22
|
+
// - A commit made after a stop's durable answer clears that answer (`instafy-stopped-clean` in the checkout's git
|
|
23
|
+
// folder), as the origin does whenever it takes the workspace, so the checkout is kept for the next start to publish.
|
|
24
|
+
//
|
|
25
|
+
// Only the backend's own writes are committed: a path whose content is no longer what the backend wrote (a person,
|
|
26
|
+
// a turn or anything else changed it since) is left alone, so work nobody saved is still caught as unsaved work.
|
|
27
|
+
// Ignored paths are never added.
|
|
28
|
+
//
|
|
29
|
+
// Limit: a stop does not ask the backend to save first. A click in the last few seconds before a machine stops (up to
|
|
30
|
+
// SAVE_MAX_DELAY_MS while clicks keep coming) can still be kept by that stop as unsaved work.
|
|
31
|
+
//
|
|
32
|
+
// Zero dependencies, Node 18+, and `perl` for the lock (on every runtime image and on macOS).
|
|
33
|
+
import { execFile, spawnSync } from "node:child_process";
|
|
34
|
+
import { createHash } from "node:crypto";
|
|
35
|
+
import fs from "node:fs";
|
|
36
|
+
import os from "node:os";
|
|
37
|
+
import path from "node:path";
|
|
38
|
+
import { fileURLToPath } from "node:url";
|
|
39
|
+
|
|
40
|
+
export const SAVE_DELAY_MS = 2_000;
|
|
41
|
+
export const SAVE_MAX_DELAY_MS = 10_000;
|
|
42
|
+
const RETRY_MS = 5_000;
|
|
43
|
+
const CHILD_TIMEOUT_MS = 30_000;
|
|
44
|
+
const GIT_TIMEOUT_MS = 20_000;
|
|
45
|
+
const MAX_NOTED = 500;
|
|
46
|
+
const SAVE_ARG = "--instafy-save-locked";
|
|
47
|
+
const BUSY_EXIT = 75;
|
|
48
|
+
const LOCK_FILE = "origin-apply.lock";
|
|
49
|
+
const CLEAN_STOP_MARKER = "instafy-stopped-clean";
|
|
50
|
+
const SAVE_MESSAGE = (label) =>
|
|
51
|
+
`Save what ${label} recorded\n\nData files a site's own backend wrote for its visitors' clicks, saved by that backend.\n\nInstafy-Saved-By: site backend\n`;
|
|
52
|
+
// Only what git needs to run; tokens and every GIT_* variable stay out.
|
|
53
|
+
const ENV_KEEP = ["PATH", "LANG", "LC_ALL", "LC_CTYPE", "TZ", "TMPDIR", "SYSTEMROOT"];
|
|
54
|
+
const GIT_SAFETY = ["-c", "core.hooksPath=/dev/null", "-c", "core.fsmonitor=false", "-c", "commit.gpgSign=false", "-c", "core.quotePath=false"];
|
|
55
|
+
// Takes the workspace lock without waiting (exit 75 when it is held), keeps it open for the command and its children
|
|
56
|
+
// ($^F: not closed on exec), and runs the command.
|
|
57
|
+
const LOCKER = [
|
|
58
|
+
"use Fcntl qw(:flock O_RDWR O_CREAT O_NOFOLLOW);",
|
|
59
|
+
"$^F = 1023;",
|
|
60
|
+
"my $p = shift @ARGV;",
|
|
61
|
+
"sysopen(my $fh, $p, O_RDWR | O_CREAT | O_NOFOLLOW, 0600) or exit 76;",
|
|
62
|
+
`flock($fh, LOCK_EX | LOCK_NB) or exit ${BUSY_EXIT};`,
|
|
63
|
+
"my $c = system { $ARGV[0] } @ARGV;",
|
|
64
|
+
"exit($c == -1 ? 127 : ($c & 127) ? 128 + ($c & 127) : $c >> 8);",
|
|
65
|
+
].join(" ");
|
|
66
|
+
|
|
67
|
+
/** The Instafy checkout a folder belongs to: the nearest `.instafy/.git` above it (or in it), else null. */
|
|
68
|
+
export function findSpaceCheckout(startDir) {
|
|
69
|
+
let dir;
|
|
70
|
+
try {
|
|
71
|
+
dir = fs.realpathSync(path.resolve(startDir));
|
|
72
|
+
} catch {
|
|
73
|
+
return null;
|
|
74
|
+
}
|
|
75
|
+
for (;;) {
|
|
76
|
+
const gitDir = path.join(dir, ".instafy", ".git");
|
|
77
|
+
try {
|
|
78
|
+
if (fs.statSync(path.join(gitDir, "HEAD")).isFile()) return { root: dir, gitDir };
|
|
79
|
+
} catch {
|
|
80
|
+
// keep looking
|
|
81
|
+
}
|
|
82
|
+
const parent = path.dirname(dir);
|
|
83
|
+
if (parent === dir) return null;
|
|
84
|
+
dir = parent;
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
function safeEnv(extra = {}) {
|
|
89
|
+
const env = {};
|
|
90
|
+
for (const key of ENV_KEEP) if (process.env[key] !== undefined) env[key] = process.env[key];
|
|
91
|
+
return { ...env, GIT_TERMINAL_PROMPT: "0", GIT_CONFIG_NOSYSTEM: "1", GIT_CONFIG_GLOBAL: "/dev/null", ...extra };
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
const sha256 = (bytes) => createHash("sha256").update(bytes).digest("hex");
|
|
95
|
+
const LOCK_RE = /index\.lock|cannot lock ref|but expected|Another git process|Unable to create .*\.lock/i;
|
|
96
|
+
|
|
97
|
+
export class SaveOwner {
|
|
98
|
+
/**
|
|
99
|
+
* @param {object} options
|
|
100
|
+
* @param {{root: string, gitDir: string}} options.checkout
|
|
101
|
+
* @param {string} options.label who wrote the files, for the commit message (a site folder)
|
|
102
|
+
* @param {(level: string, text: string) => void} [options.log]
|
|
103
|
+
* @param {number} [options.delayMs] quiet time after the last write before committing
|
|
104
|
+
* @param {number} [options.maxDelayMs] longest a write waits while writes keep coming
|
|
105
|
+
*/
|
|
106
|
+
constructor({ checkout, label, log = () => {}, delayMs = SAVE_DELAY_MS, maxDelayMs = SAVE_MAX_DELAY_MS }) {
|
|
107
|
+
this.root = checkout.root;
|
|
108
|
+
this.gitDir = checkout.gitDir;
|
|
109
|
+
this.label = String(label || "a site").replace(/[\r\n]+/g, " ").slice(0, 120);
|
|
110
|
+
this.log = log;
|
|
111
|
+
this.delayMs = delayMs;
|
|
112
|
+
this.maxDelayMs = maxDelayMs;
|
|
113
|
+
/** rel path -> the exact bytes this backend wrote last */
|
|
114
|
+
this.noted = new Map();
|
|
115
|
+
/** index entries committed but not yet in the checkout's index (it was busy) */
|
|
116
|
+
this.indexPending = [];
|
|
117
|
+
this.timer = null;
|
|
118
|
+
this.firstPendingAt = null;
|
|
119
|
+
this.running = null;
|
|
120
|
+
this.closed = false;
|
|
121
|
+
this.commits = 0;
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
/** Inside the checkout's work tree (and not in its git folder): the posix path relative to the root, else null. */
|
|
125
|
+
relative(file) {
|
|
126
|
+
let real;
|
|
127
|
+
try {
|
|
128
|
+
real = path.join(fs.realpathSync(path.dirname(path.resolve(file))), path.basename(file));
|
|
129
|
+
} catch {
|
|
130
|
+
return null;
|
|
131
|
+
}
|
|
132
|
+
const rel = path.relative(this.root, real);
|
|
133
|
+
if (!rel || rel.startsWith("..") || path.isAbsolute(rel)) return null;
|
|
134
|
+
const posix = rel.split(path.sep).join("/");
|
|
135
|
+
if (posix === ".instafy" || posix.startsWith(".instafy/")) return null;
|
|
136
|
+
return posix;
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
/** Record that this backend wrote `content` to `file`, and commit it soon. */
|
|
140
|
+
note(file, content) {
|
|
141
|
+
if (this.closed) return;
|
|
142
|
+
const rel = this.relative(file);
|
|
143
|
+
if (!rel) return;
|
|
144
|
+
if (!this.noted.has(rel) && this.noted.size >= MAX_NOTED) return;
|
|
145
|
+
this.noted.set(rel, Buffer.isBuffer(content) ? content : Buffer.from(String(content)));
|
|
146
|
+
this.schedule(this.delayMs);
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
schedule(ms) {
|
|
150
|
+
if (this.closed) return;
|
|
151
|
+
const now = Date.now();
|
|
152
|
+
this.firstPendingAt ??= now;
|
|
153
|
+
const wait = Math.max(0, Math.min(ms, this.firstPendingAt + this.maxDelayMs - now));
|
|
154
|
+
if (this.timer) clearTimeout(this.timer);
|
|
155
|
+
this.timer = setTimeout(() => {
|
|
156
|
+
this.timer = null;
|
|
157
|
+
void this.flush();
|
|
158
|
+
}, wait);
|
|
159
|
+
this.timer.unref?.();
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
/** The noted paths whose file still holds exactly what this backend wrote. */
|
|
163
|
+
ownedPaths() {
|
|
164
|
+
const owned = [];
|
|
165
|
+
for (const [rel, bytes] of this.noted) {
|
|
166
|
+
let current;
|
|
167
|
+
try {
|
|
168
|
+
current = fs.readFileSync(path.join(this.root, ...rel.split("/")));
|
|
169
|
+
} catch {
|
|
170
|
+
this.noted.delete(rel);
|
|
171
|
+
continue;
|
|
172
|
+
}
|
|
173
|
+
if (current.equals(bytes)) owned.push({ rel, bytes });
|
|
174
|
+
else this.noted.delete(rel);
|
|
175
|
+
}
|
|
176
|
+
return owned;
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
/** Commit what this backend wrote and still owns. Resolves to {committed, paths} or {skipped, reason}. */
|
|
180
|
+
flush({ sync = false } = {}) {
|
|
181
|
+
if (this.timer) {
|
|
182
|
+
clearTimeout(this.timer);
|
|
183
|
+
this.timer = null;
|
|
184
|
+
}
|
|
185
|
+
if (sync) return this.commitOwned(true);
|
|
186
|
+
if (this.running) {
|
|
187
|
+
// One commit at a time; a write during it waits for the next round.
|
|
188
|
+
return this.running.then(() => (this.noted.size ? this.flush() : { skipped: true, reason: "nothing to save" }));
|
|
189
|
+
}
|
|
190
|
+
this.running = Promise.resolve(this.commitOwned(false)).finally(() => {
|
|
191
|
+
this.running = null;
|
|
192
|
+
});
|
|
193
|
+
return this.running;
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
commitOwned(sync) {
|
|
197
|
+
const owned = this.ownedPaths();
|
|
198
|
+
this.firstPendingAt = null;
|
|
199
|
+
if (owned.length === 0 && this.indexPending.length === 0) {
|
|
200
|
+
const nothing = { skipped: true, reason: "nothing to save" };
|
|
201
|
+
return sync ? nothing : Promise.resolve(nothing);
|
|
202
|
+
}
|
|
203
|
+
const payload = JSON.stringify({
|
|
204
|
+
root: this.root,
|
|
205
|
+
gitDir: this.gitDir,
|
|
206
|
+
message: SAVE_MESSAGE(this.label),
|
|
207
|
+
entries: owned.map((entry) => ({ rel: entry.rel, sha256: sha256(entry.bytes) })),
|
|
208
|
+
pending: this.indexPending,
|
|
209
|
+
});
|
|
210
|
+
const finish = (outcome) => {
|
|
211
|
+
const result = parseOutcome(outcome);
|
|
212
|
+
if (Array.isArray(result.indexPending)) this.indexPending = result.indexPending;
|
|
213
|
+
if (result.retry) {
|
|
214
|
+
if (!this.closed) {
|
|
215
|
+
for (const entry of owned) if (!this.noted.has(entry.rel)) this.noted.set(entry.rel, entry.bytes);
|
|
216
|
+
this.schedule(RETRY_MS);
|
|
217
|
+
}
|
|
218
|
+
this.log("info", `save: the space is busy, trying again (${result.reason})`);
|
|
219
|
+
return { skipped: true, reason: result.reason };
|
|
220
|
+
}
|
|
221
|
+
for (const entry of owned) {
|
|
222
|
+
const now = this.noted.get(entry.rel);
|
|
223
|
+
if (now && now.equals(entry.bytes)) this.noted.delete(entry.rel);
|
|
224
|
+
}
|
|
225
|
+
if (this.indexPending.length && !this.closed) this.schedule(RETRY_MS);
|
|
226
|
+
if (result.committed) this.commits += 1;
|
|
227
|
+
if (result.error) this.log("warn", `save: ${result.error}`);
|
|
228
|
+
delete result.indexPending;
|
|
229
|
+
return result;
|
|
230
|
+
};
|
|
231
|
+
const lock = path.join(this.root, ".instafy", LOCK_FILE);
|
|
232
|
+
const args = ["-e", LOCKER, lock, process.execPath, fileURLToPath(import.meta.url), SAVE_ARG];
|
|
233
|
+
const options = { cwd: this.root, env: safeEnv(), timeout: CHILD_TIMEOUT_MS, maxBuffer: 4 * 1024 * 1024, encoding: "utf8" };
|
|
234
|
+
if (sync) {
|
|
235
|
+
const ran = spawnSync("perl", args, { ...options, input: payload });
|
|
236
|
+
return finish({ code: ran.error ? null : ran.status, stdout: ran.stdout, stderr: ran.stderr, error: ran.error });
|
|
237
|
+
}
|
|
238
|
+
return new Promise((resolve) => {
|
|
239
|
+
const child = execFile("perl", args, options, (error, stdout, stderr) => {
|
|
240
|
+
resolve(finish({ code: error ? (typeof error.code === "number" ? error.code : null) : 0, stdout, stderr, error: error && typeof error.code !== "number" ? error : null }));
|
|
241
|
+
});
|
|
242
|
+
child.stdin.on("error", () => {});
|
|
243
|
+
child.stdin.end(payload);
|
|
244
|
+
});
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
/** Commit what is left, synchronously (a stopping process), and stop. A busy space is waited for briefly. */
|
|
248
|
+
close() {
|
|
249
|
+
if (this.closed) return { skipped: true, reason: "closed" };
|
|
250
|
+
if (this.timer) clearTimeout(this.timer);
|
|
251
|
+
this.timer = null;
|
|
252
|
+
let result = { skipped: true, reason: "nothing to save" };
|
|
253
|
+
const until = Date.now() + 3_000;
|
|
254
|
+
while (this.noted.size || this.indexPending.length) {
|
|
255
|
+
result = this.flush({ sync: true });
|
|
256
|
+
if (!(result.skipped && /busy/.test(result.reason || "")) || Date.now() > until) break;
|
|
257
|
+
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 200);
|
|
258
|
+
}
|
|
259
|
+
this.closed = true;
|
|
260
|
+
if (this.timer) clearTimeout(this.timer);
|
|
261
|
+
this.timer = null;
|
|
262
|
+
return result;
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
function parseOutcome({ code, stdout, stderr, error }) {
|
|
267
|
+
if (code === BUSY_EXIT) return { retry: true, reason: "workspace busy" };
|
|
268
|
+
if (error) return { error: error.code === "ENOENT" ? "perl is missing, so the workspace lock cannot be taken" : String(error.message || error) };
|
|
269
|
+
try {
|
|
270
|
+
const parsed = JSON.parse(String(stdout || "").trim().split("\n").pop() || "null");
|
|
271
|
+
if (parsed && typeof parsed === "object") return parsed;
|
|
272
|
+
} catch {
|
|
273
|
+
// fall through
|
|
274
|
+
}
|
|
275
|
+
return { error: `the save stopped (exit ${code}): ${String(stderr || "").trim().split("\n")[0]}` };
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
// ---------------------------------------------------------------------------------------------------------------------
|
|
279
|
+
// The commit itself: runs in a child process that holds the workspace lock (see LOCKER), synchronously.
|
|
280
|
+
// ---------------------------------------------------------------------------------------------------------------------
|
|
281
|
+
|
|
282
|
+
function lockedSave(input) {
|
|
283
|
+
const { root, gitDir, message, entries = [], pending = [] } = input;
|
|
284
|
+
const attrs = {};
|
|
285
|
+
const git = (args, { stdin, index } = {}) => {
|
|
286
|
+
const ran = spawnSync("git", ["--git-dir", gitDir, "--work-tree", root, ...GIT_SAFETY, ...args], {
|
|
287
|
+
cwd: root,
|
|
288
|
+
env: safeEnv({ ...attrs, ...(index ? { GIT_INDEX_FILE: index } : {}) }),
|
|
289
|
+
input: stdin,
|
|
290
|
+
timeout: GIT_TIMEOUT_MS,
|
|
291
|
+
maxBuffer: 64 * 1024 * 1024,
|
|
292
|
+
});
|
|
293
|
+
return { code: ran.status ?? 1, stdout: ran.stdout ? ran.stdout.toString("utf8") : "", stderr: ran.stderr ? ran.stderr.toString("utf8") : String(ran.error?.message || "") };
|
|
294
|
+
};
|
|
295
|
+
const failed = (step, result) =>
|
|
296
|
+
LOCK_RE.test(result.stderr) ? { retry: true, reason: `${step}: busy` } : { error: `${step} failed: ${(result.stderr || "").trim().split("\n")[0] || `exit ${result.code}`}` };
|
|
297
|
+
const headOid = () => {
|
|
298
|
+
const head = git(["rev-parse", "-q", "--verify", "HEAD^{commit}"]);
|
|
299
|
+
return head.code === 0 ? head.stdout.trim() : null;
|
|
300
|
+
};
|
|
301
|
+
const treeEntry = (commit, rel) => {
|
|
302
|
+
if (!commit) return null;
|
|
303
|
+
const listed = git(["ls-tree", "-z", commit, "--", rel]).stdout.split("\0")[0];
|
|
304
|
+
const match = /^(\d+) blob ([0-9a-f]+)\t/.exec(listed);
|
|
305
|
+
return match ? { mode: match[1], oid: match[2] } : null;
|
|
306
|
+
};
|
|
307
|
+
// The checkout's index takes the committed entries (no filters, no stat data: git fills that in on its next look).
|
|
308
|
+
const syncIndex = (list) => {
|
|
309
|
+
if (!list.length) return [];
|
|
310
|
+
const head = headOid();
|
|
311
|
+
const still = list.filter((entry) => {
|
|
312
|
+
const now = treeEntry(head, entry.rel);
|
|
313
|
+
return now && now.oid === entry.oid;
|
|
314
|
+
});
|
|
315
|
+
if (!still.length) return [];
|
|
316
|
+
const info = still.map((entry) => `${entry.mode} ${entry.oid}\t${entry.rel}\0`).join("");
|
|
317
|
+
for (let attempt = 0; attempt < 20; attempt += 1) {
|
|
318
|
+
const updated = git(["update-index", "-z", "--index-info"], { stdin: info });
|
|
319
|
+
if (updated.code === 0) return [];
|
|
320
|
+
if (!LOCK_RE.test(updated.stderr)) break;
|
|
321
|
+
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 50);
|
|
322
|
+
}
|
|
323
|
+
return still;
|
|
324
|
+
};
|
|
325
|
+
|
|
326
|
+
// Writing the checkout's index makes git look again at files it cannot tell apart by time, through their
|
|
327
|
+
// attributes' filters. Attributes come from an empty tree instead (git 2.40+), so no filter of the workspace's runs.
|
|
328
|
+
const empty = git(["mktree"], { stdin: "" });
|
|
329
|
+
if (empty.code !== 0) return { error: `mktree failed: ${empty.stderr.trim().split("\n")[0]}` };
|
|
330
|
+
attrs.GIT_ATTR_SOURCE = empty.stdout.trim();
|
|
331
|
+
|
|
332
|
+
let indexPending = syncIndex(pending);
|
|
333
|
+
const done = (result) => ({ ...result, indexPending });
|
|
334
|
+
if (fs.existsSync(path.join(gitDir, "index.lock"))) return done({ retry: true, reason: "index: busy" });
|
|
335
|
+
|
|
336
|
+
// What still holds the backend's bytes, read once: these bytes are what gets committed.
|
|
337
|
+
const files = [];
|
|
338
|
+
for (const entry of entries) {
|
|
339
|
+
let bytes;
|
|
340
|
+
try {
|
|
341
|
+
const file = path.join(root, ...entry.rel.split("/"));
|
|
342
|
+
if (!fs.lstatSync(file).isFile()) continue;
|
|
343
|
+
bytes = fs.readFileSync(file);
|
|
344
|
+
} catch {
|
|
345
|
+
continue;
|
|
346
|
+
}
|
|
347
|
+
if (sha256(bytes) === entry.sha256) files.push({ rel: entry.rel, bytes });
|
|
348
|
+
}
|
|
349
|
+
if (!files.length) return done({ skipped: true, reason: entries.length ? "changed since" : "nothing to save" });
|
|
350
|
+
|
|
351
|
+
// Ignored paths stay out (the repository chose not to keep them). Tracked files are never reported as ignored.
|
|
352
|
+
const ignored = git(["check-ignore", "-z", "--stdin"], { stdin: files.map((file) => `${file.rel}\0`).join("") });
|
|
353
|
+
const skip = new Set(ignored.code === 0 ? ignored.stdout.split("\0").filter(Boolean) : []);
|
|
354
|
+
const keep = files.filter((file) => !skip.has(file.rel));
|
|
355
|
+
if (!keep.length) return done({ skipped: true, reason: "ignored by the repository" });
|
|
356
|
+
|
|
357
|
+
const head = headOid();
|
|
358
|
+
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "instafy-save-"));
|
|
359
|
+
try {
|
|
360
|
+
const index = path.join(tmp, "index");
|
|
361
|
+
const read = head ? git(["read-tree", head], { index }) : git(["read-tree", "--empty"], { index });
|
|
362
|
+
if (read.code !== 0) return done(failed("read-tree", read));
|
|
363
|
+
const committed = [];
|
|
364
|
+
for (const file of keep) {
|
|
365
|
+
const hashed = git(["hash-object", "-w", "--no-filters", "--stdin"], { stdin: file.bytes });
|
|
366
|
+
if (hashed.code !== 0) return done(failed("hash-object", hashed));
|
|
367
|
+
const before = treeEntry(head, file.rel);
|
|
368
|
+
committed.push({ rel: file.rel, mode: before?.mode === "100755" ? "100755" : "100644", oid: hashed.stdout.trim() });
|
|
369
|
+
}
|
|
370
|
+
const info = committed.map((entry) => `${entry.mode} ${entry.oid}\t${entry.rel}\0`).join("");
|
|
371
|
+
const staged = git(["update-index", "-z", "--index-info"], { stdin: info, index });
|
|
372
|
+
if (staged.code !== 0) return done(failed("update-index", staged));
|
|
373
|
+
const tree = git(["write-tree"], { index });
|
|
374
|
+
if (tree.code !== 0) return done(failed("write-tree", tree));
|
|
375
|
+
const treeOid = tree.stdout.trim();
|
|
376
|
+
if (head && git(["rev-parse", `${head}^{tree}`]).stdout.trim() === treeOid) {
|
|
377
|
+
indexPending = syncIndex([...indexPending, ...committed]);
|
|
378
|
+
return done({ skipped: true, reason: "already saved" });
|
|
379
|
+
}
|
|
380
|
+
const email = git(["config", "--get", "user.email"]);
|
|
381
|
+
const identity =
|
|
382
|
+
email.code === 0 && email.stdout.trim()
|
|
383
|
+
? {}
|
|
384
|
+
: { GIT_AUTHOR_NAME: "Instafy site", GIT_AUTHOR_EMAIL: "site@instafy.dev", GIT_COMMITTER_NAME: "Instafy site", GIT_COMMITTER_EMAIL: "site@instafy.dev" };
|
|
385
|
+
const commitRun = spawnSync("git", ["--git-dir", gitDir, "--work-tree", root, ...GIT_SAFETY, "commit-tree", treeOid, ...(head ? ["-p", head] : []), "-F", "-"], {
|
|
386
|
+
cwd: root,
|
|
387
|
+
env: safeEnv(identity),
|
|
388
|
+
input: message,
|
|
389
|
+
timeout: GIT_TIMEOUT_MS,
|
|
390
|
+
});
|
|
391
|
+
if (commitRun.status !== 0) return done(failed("commit-tree", { code: commitRun.status, stderr: String(commitRun.stderr || "") }));
|
|
392
|
+
const commit = String(commitRun.stdout).trim();
|
|
393
|
+
// HEAD moves only from the commit this save started from.
|
|
394
|
+
const moved = git(["update-ref", "-m", "instafy: a site backend saved its writes", "HEAD", commit, head || ""]);
|
|
395
|
+
if (moved.code !== 0) return done(failed("update-ref", moved));
|
|
396
|
+
try {
|
|
397
|
+
fs.rmSync(path.join(gitDir, CLEAN_STOP_MARKER), { force: true });
|
|
398
|
+
} catch {
|
|
399
|
+
// the origin's next stop decides again
|
|
400
|
+
}
|
|
401
|
+
indexPending = syncIndex([...indexPending, ...committed]);
|
|
402
|
+
return done({ committed: true, commit, paths: committed.map((entry) => entry.rel) });
|
|
403
|
+
} finally {
|
|
404
|
+
fs.rmSync(tmp, { recursive: true, force: true });
|
|
405
|
+
}
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
function isMain() {
|
|
409
|
+
try {
|
|
410
|
+
return process.argv[2] === SAVE_ARG && fs.realpathSync(process.argv[1]) === fs.realpathSync(fileURLToPath(import.meta.url));
|
|
411
|
+
} catch {
|
|
412
|
+
return false;
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
|
|
416
|
+
if (isMain()) {
|
|
417
|
+
let result;
|
|
418
|
+
try {
|
|
419
|
+
result = lockedSave(JSON.parse(fs.readFileSync(0, "utf8")));
|
|
420
|
+
} catch (error) {
|
|
421
|
+
result = { error: `save failed: ${error?.message || error}` };
|
|
422
|
+
}
|
|
423
|
+
process.stdout.write(`${JSON.stringify(result)}\n`);
|
|
424
|
+
}
|
|
425
|
+
|
|
426
|
+
const owners = new Map();
|
|
427
|
+
|
|
428
|
+
/**
|
|
429
|
+
* Make a backend the save owner of the files it writes under `siteDir`, when the site is in an Instafy space's
|
|
430
|
+
* checkout. Returns the owner, or null outside a space (a plain folder or a person's own repository: their git is
|
|
431
|
+
* theirs, and nothing is committed there).
|
|
432
|
+
*/
|
|
433
|
+
export function ownSiteWrites(siteDir, { label, log, delayMs, maxDelayMs } = {}) {
|
|
434
|
+
const checkout = findSpaceCheckout(siteDir);
|
|
435
|
+
if (!checkout) return null;
|
|
436
|
+
let real;
|
|
437
|
+
try {
|
|
438
|
+
real = fs.realpathSync(siteDir);
|
|
439
|
+
} catch {
|
|
440
|
+
return null;
|
|
441
|
+
}
|
|
442
|
+
const existing = owners.get(real);
|
|
443
|
+
if (existing && !existing.closed) return existing;
|
|
444
|
+
const owner = new SaveOwner({ checkout, label: label || path.relative(checkout.root, real) || path.basename(real), log, delayMs, maxDelayMs });
|
|
445
|
+
owner.siteDir = real;
|
|
446
|
+
owners.set(real, owner);
|
|
447
|
+
return owner;
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
/** The owner of a file a backend just wrote: the one whose site folder holds it. */
|
|
451
|
+
function ownerOf(file) {
|
|
452
|
+
let dir;
|
|
453
|
+
try {
|
|
454
|
+
dir = fs.realpathSync(path.dirname(path.resolve(file)));
|
|
455
|
+
} catch {
|
|
456
|
+
return null;
|
|
457
|
+
}
|
|
458
|
+
let best = null;
|
|
459
|
+
for (const [siteDir, owner] of owners) {
|
|
460
|
+
if (owner.closed) continue;
|
|
461
|
+
if ((dir === siteDir || dir.startsWith(siteDir + path.sep)) && (!best || siteDir.length > best.siteDir.length)) best = owner;
|
|
462
|
+
}
|
|
463
|
+
return best;
|
|
464
|
+
}
|
|
465
|
+
|
|
466
|
+
/**
|
|
467
|
+
* Called by a backend's write path after it wrote `content` (bytes, text, or a function that reads them) to `file`.
|
|
468
|
+
* No owner in this process (no running server, or not in a space): nothing happens.
|
|
469
|
+
*/
|
|
470
|
+
export function noteSiteWrite(file, content) {
|
|
471
|
+
if (owners.size === 0) return;
|
|
472
|
+
const owner = ownerOf(file);
|
|
473
|
+
if (!owner) return;
|
|
474
|
+
try {
|
|
475
|
+
owner.note(file, typeof content === "function" ? content() : content);
|
|
476
|
+
} catch {
|
|
477
|
+
// a write the owner cannot read back is left to the next save
|
|
478
|
+
}
|
|
479
|
+
}
|
|
480
|
+
|
|
481
|
+
/** Stop owning a site's writes, committing what is left. */
|
|
482
|
+
export function releaseSiteWrites(siteDir) {
|
|
483
|
+
let real;
|
|
484
|
+
try {
|
|
485
|
+
real = fs.realpathSync(siteDir);
|
|
486
|
+
} catch {
|
|
487
|
+
real = path.resolve(siteDir);
|
|
488
|
+
}
|
|
489
|
+
const owner = owners.get(real);
|
|
490
|
+
owners.delete(real);
|
|
491
|
+
return owner ? owner.close() : null;
|
|
492
|
+
}
|