@hotmeshio/long-tail 0.28.0 → 0.28.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/api/files.d.ts +5 -0
- package/build/api/files.js +19 -2
- package/build/api/knowledge.d.ts +3 -1
- package/build/lib/db/schemas/042_knowledge_edition_labels.sql +9 -0
- package/build/lib/events/publish.js +3 -2
- package/build/lib/machines/line.d.ts +3 -0
- package/build/lib/machines/line.js +7 -1
- package/build/lib/machines/protocol/codec.d.ts +6 -0
- package/build/lib/machines/protocol/codec.js +1 -1
- package/build/lib/machines/protocol/contract.d.ts +2 -0
- package/build/lib/machines/protocol/contract.js +12 -7
- package/build/lib/machines/site/mosquitto-conf.d.ts +7 -3
- package/build/lib/machines/site/mosquitto-conf.js +3 -8
- package/build/lib/machines/site/site-conf.d.ts +4 -1
- package/build/lib/machines/site/site-conf.js +5 -1
- package/build/lib/machines/transport/nats-bus.d.ts +4 -0
- package/build/lib/machines/transport/stream-bus.d.ts +7 -1
- package/build/lib/machines/transport/stream-bus.js +20 -6
- package/build/lib/machines/transport/streams.d.ts +11 -2
- package/build/lib/machines/transport/streams.js +33 -5
- package/build/lib/messaging/memory.js +10 -3
- package/build/lib/messaging/nats/permissions.js +1 -0
- package/build/lib/messaging/nats/streams.d.ts +21 -4
- package/build/lib/messaging/nats/streams.js +71 -10
- package/build/lib/storage/local.d.ts +2 -0
- package/build/lib/storage/local.js +3 -0
- package/build/lib/storage/s3.d.ts +3 -2
- package/build/lib/storage/s3.js +6 -1
- package/build/lib/storage/types.d.ts +7 -1
- package/build/routes/file-browser.js +3 -1
- package/build/routes/knowledge.js +6 -5
- package/build/routes/machines.js +33 -0
- package/build/services/knowledge/index.d.ts +4 -2
- package/build/services/knowledge/index.js +8 -2
- package/build/services/knowledge/lookup-cache.d.ts +8 -0
- package/build/services/knowledge/lookup-cache.js +30 -3
- package/build/services/knowledge/lookup-refs.js +1 -1
- package/build/services/knowledge/sql.d.ts +4 -2
- package/build/services/knowledge/sql.js +8 -3
- package/build/services/machines/config.d.ts +2 -0
- package/build/services/machines/config.js +1 -0
- package/build/services/machines/contracts.d.ts +18 -0
- package/build/services/machines/contracts.js +15 -1
- package/build/services/machines/forms.d.ts +26 -1
- package/build/services/machines/forms.js +17 -1
- package/build/services/machines/gateway/actions.d.ts +13 -1
- package/build/services/machines/gateway/admission.d.ts +15 -1
- package/build/services/machines/gateway/admission.js +47 -0
- package/build/services/machines/gateway/api-escalations.js +4 -0
- package/build/services/machines/gateway/dispatcher.d.ts +9 -4
- package/build/services/machines/gateway/dispatcher.js +42 -14
- package/build/services/machines/gateway/executor.d.ts +2 -0
- package/build/services/machines/gateway/executor.js +18 -2
- package/build/services/machines/gateway/gateway.d.ts +5 -0
- package/build/services/machines/gateway/gateway.js +11 -3
- package/build/services/machines/gateway/log.d.ts +3 -0
- package/build/services/machines/gateway/log.js +3 -0
- package/build/services/machines/gateway/lt-http.js +11 -2
- package/build/services/machines/gateway/pacing.js +1 -1
- package/build/services/machines/gateway/plan-machine.js +26 -11
- package/build/services/machines/gateway/plan-shared.d.ts +5 -3
- package/build/services/machines/gateway/plan-shared.js +6 -4
- package/build/services/machines/gateway/reply-waiter.d.ts +2 -0
- package/build/services/machines/gateway/state.d.ts +24 -1
- package/build/services/machines/gateway/state.js +67 -6
- package/build/services/machines/gateway/wire-window.d.ts +21 -0
- package/build/services/machines/gateway/wire-window.js +61 -0
- package/build/services/machines/gateway/wire.js +1 -0
- package/build/services/machines/leadership.d.ts +2 -0
- package/build/services/machines/leadership.js +7 -1
- package/build/services/machines/roles.d.ts +5 -4
- package/build/services/machines/roles.js +13 -4
- package/build/services/machines/select.d.ts +3 -2
- package/build/services/machines/select.js +4 -3
- package/build/services/machines/surface.d.ts +10 -1
- package/build/services/machines/surface.js +27 -5
- package/build/system/activities/knowledge.d.ts +4 -0
- package/build/system/activities/knowledge.js +15 -0
- package/build/system/activities/sql.d.ts +2 -0
- package/build/system/activities/sql.js +6 -1
- package/build/system/mcp-servers/knowledge.js +4 -2
- package/build/tsconfig.tsbuildinfo +1 -1
- package/build/types/escalation.d.ts +9 -6
- package/build/types/escalation.js +8 -2
- package/build/types/machines-view.d.ts +16 -0
- package/build/types/machines.d.ts +1 -0
- package/build/types/messaging.d.ts +7 -1
- package/build/types/startup.d.ts +2 -0
- package/docs/dashboard.md +1 -1
- package/docs/hitl/lookups.md +6 -4
- package/docs/machines.md +26 -17
- package/package.json +1 -1
|
@@ -36,7 +36,10 @@ function planMachineMessage(state, parsed, env, dispatchTtlMs) {
|
|
|
36
36
|
function planByKind(state, m, parsed, env, b, ttl) {
|
|
37
37
|
switch (parsed.kind) {
|
|
38
38
|
case protocol_1.MESSAGE_KIND.REPLY:
|
|
39
|
-
return ack([{
|
|
39
|
+
return ack([{
|
|
40
|
+
kind: 'settleReply', cmdId: b.cmdId, status: b.status, reason: b.reason,
|
|
41
|
+
...(typeof b.code === 'string' ? { code: b.code } : {}), ...(b.preparing === true ? { preparing: true } : {}),
|
|
42
|
+
}]);
|
|
40
43
|
case protocol_1.MESSAGE_KIND.JOB:
|
|
41
44
|
return planJob(state, m, b, env?.at, ttl);
|
|
42
45
|
case protocol_1.MESSAGE_KIND.ALARM:
|
|
@@ -191,7 +194,9 @@ function planAlarm(state, parsed, alarmId, env) {
|
|
|
191
194
|
payload: { decision: contracts_1.DECISION.CLEARED, clearedBy: b.clearedBy ?? null, note: `cleared at ${m}${b.clearedBy ? ` by ${b.clearedBy}` : ''}` },
|
|
192
195
|
}]);
|
|
193
196
|
}
|
|
194
|
-
|
|
197
|
+
// A stopping level needs a person; so does any alarm the machine flags, at the level it said.
|
|
198
|
+
const stopping = protocol_1.STOPPING_LEVELS.includes(b.level);
|
|
199
|
+
if (!stopping && b.needsPerson !== true)
|
|
195
200
|
return ack();
|
|
196
201
|
const known = state.alarms.get(key);
|
|
197
202
|
if (known && (!known.raisedAt || !b.raisedAt || known.raisedAt === b.raisedAt))
|
|
@@ -201,7 +206,7 @@ function planAlarm(state, parsed, alarmId, env) {
|
|
|
201
206
|
: [];
|
|
202
207
|
const track = { machineId: m, alarmId, raisedAt: b.raisedAt, level: b.level, open: true };
|
|
203
208
|
state.alarms.set(key, track);
|
|
204
|
-
return ack([...superseded, ...(0, lifecycle_1.notePause)(state, track, b), {
|
|
209
|
+
return ack([...superseded, ...(stopping ? (0, lifecycle_1.notePause)(state, track, b) : []), {
|
|
205
210
|
kind: 'createAlarmRow', machineId: m, alarmId,
|
|
206
211
|
row: {
|
|
207
212
|
type: contracts_1.ROW_TYPE.ALARM, subtype: b.level, role: contracts_1.LINE_ROLE.OPERATOR, priority: b.level === protocol_1.LEVEL.EMERGENCY ? 1 : 2,
|
|
@@ -254,27 +259,37 @@ function planPresence(state, m, b) {
|
|
|
254
259
|
}
|
|
255
260
|
return ack(actions);
|
|
256
261
|
}
|
|
257
|
-
/**
|
|
262
|
+
/**
|
|
263
|
+
* A new boot id means the machine restarted and lost its queue: send its open
|
|
264
|
+
* dispatches again, and the jobs it had ready for units it still holds, from
|
|
265
|
+
* the rows that sent them. A lost job nothing is remembered for is logged.
|
|
266
|
+
*/
|
|
258
267
|
function planSnapshot(state, m, b, ttl) {
|
|
259
268
|
const link = state.link(m);
|
|
260
269
|
if (b.software)
|
|
261
270
|
link.software = b.software;
|
|
262
271
|
const jobIds = Array.isArray(b.jobs) ? b.jobs.map((j) => j?.jobId).filter((id) => typeof id === 'string') : [];
|
|
272
|
+
// A snapshot without a boot id says nothing about restarts.
|
|
273
|
+
const rebooted = typeof b.bootId === 'string' && !!link.bootId && link.bootId !== b.bootId;
|
|
274
|
+
const held = new Set(jobIds);
|
|
275
|
+
const lost = rebooted ? state.lostAfterReboot(m, held) : { resend: [], unknown: [] };
|
|
263
276
|
if (Array.isArray(b.jobs)) {
|
|
264
|
-
state.keepHeld(m, new Set(jobIds));
|
|
277
|
+
state.keepHeld(m, new Set([...jobIds, ...lost.resend.map((l) => l.jobId)]));
|
|
265
278
|
link.jobs = b.jobs;
|
|
266
279
|
}
|
|
280
|
+
if (typeof b.bootId === 'string')
|
|
281
|
+
link.bootId = b.bootId;
|
|
267
282
|
const fresh = !state.retainedAlarms && !state.readingRetained && Array.isArray(b.alarms);
|
|
268
283
|
const cleared = fresh ? alarmsNoLongerListed(state, m, b.alarms) : [];
|
|
269
284
|
const answer = { kind: 'settleSnapshot', machineId: m, jobIds };
|
|
270
|
-
|
|
271
|
-
link.bootId = b.bootId;
|
|
272
|
-
if (!previous || previous === b.bootId)
|
|
285
|
+
if (!rebooted)
|
|
273
286
|
return ack([...cleared, answer]);
|
|
274
|
-
const held = new Set(jobIds);
|
|
275
287
|
const missing = (r) => (0, jobs_1.dispatchJobs)(r.jobId, r.units, state.unitsPerDispatch).some((j) => !held.has(j.jobId));
|
|
276
|
-
const
|
|
277
|
-
|
|
288
|
+
const open = state.open().filter((r) => r.toMachine === m && r.dispatch === state_1.DISPATCH_STATE.ACCEPTED && missing(r));
|
|
289
|
+
const rows = new Map(lost.resend.map((l) => [l.row.id, l.row]));
|
|
290
|
+
const again = [...rows.values()].map((r) => ({ ...(0, pacing_1.dispatchAction)(r, m, ttl), ended: true }));
|
|
291
|
+
const unknown = lost.unknown.length ? [{ kind: 'unrecoverable', machineId: m, jobs: lost.unknown }] : [];
|
|
292
|
+
return ack([...cleared, answer, ...open.map((r) => (0, pacing_1.dispatchAction)(r, m, ttl)), ...again, ...unknown]);
|
|
278
293
|
}
|
|
279
294
|
/** Alarm rows for `m` whose alarm its snapshot no longer lists: they cleared at the machine. */
|
|
280
295
|
function alarmsNoLongerListed(state, m, listed) {
|
|
@@ -5,9 +5,11 @@ import { type GatewayState, type HandoffRow } from './state';
|
|
|
5
5
|
/** A cancel for each job a handoff's units became at `machineId`, or for those `keep` leaves out. */
|
|
6
6
|
export declare function cancelJobs(state: GatewayState, row: HandoffRow, machineId: string, jobId: string, tag?: string, keep?: (job: DispatchJob) => boolean): Action[];
|
|
7
7
|
/**
|
|
8
|
-
* Withdraw a dispatch that may still sit in the receiver's queue
|
|
9
|
-
*
|
|
10
|
-
*
|
|
8
|
+
* Withdraw a dispatch that may still sit in the receiver's queue: one sent or
|
|
9
|
+
* accepted. A job that never went out (the row waiting for room, or for its
|
|
10
|
+
* units to arrive) leaves the receiver nothing to withdraw. A job some of whose
|
|
11
|
+
* units already reached the receiver is under way: it stays, and what happens
|
|
12
|
+
* to it is the workflow's next decision (a machine-command row).
|
|
11
13
|
*/
|
|
12
14
|
export declare function cancelDispatch(state: GatewayState, row: HandoffRow): Action[];
|
|
13
15
|
/**
|
|
@@ -14,13 +14,15 @@ function cancelJobs(state, row, machineId, jobId, tag = '', keep = () => false)
|
|
|
14
14
|
}));
|
|
15
15
|
}
|
|
16
16
|
/**
|
|
17
|
-
* Withdraw a dispatch that may still sit in the receiver's queue
|
|
18
|
-
*
|
|
19
|
-
*
|
|
17
|
+
* Withdraw a dispatch that may still sit in the receiver's queue: one sent or
|
|
18
|
+
* accepted. A job that never went out (the row waiting for room, or for its
|
|
19
|
+
* units to arrive) leaves the receiver nothing to withdraw. A job some of whose
|
|
20
|
+
* units already reached the receiver is under way: it stays, and what happens
|
|
21
|
+
* to it is the workflow's next decision (a machine-command row).
|
|
20
22
|
*/
|
|
21
23
|
function cancelDispatch(state, row) {
|
|
22
24
|
const to = row.toMachine;
|
|
23
|
-
if (!to || row.dispatch
|
|
25
|
+
if (!to || (row.dispatch !== state_1.DISPATCH_STATE.PENDING && row.dispatch !== state_1.DISPATCH_STATE.ACCEPTED))
|
|
24
26
|
return [];
|
|
25
27
|
return cancelJobs(state, row, to, row.jobId, '', (job) => job.units.some((u) => state.heldJob(to, u.unitId) === job.jobId));
|
|
26
28
|
}
|
|
@@ -10,6 +10,8 @@ export interface CommandReply {
|
|
|
10
10
|
reason: string;
|
|
11
11
|
/** The machine's structured code for the outcome, when its reply carried one. */
|
|
12
12
|
code?: string;
|
|
13
|
+
/** An acceptance the machine will follow with `completed` or `failed` once the job is prepared. */
|
|
14
|
+
preparing?: boolean;
|
|
13
15
|
}
|
|
14
16
|
/** The jobs a machine reports holding, queued or in work. */
|
|
15
17
|
export type HeldJobs = Set<string>;
|
|
@@ -35,6 +35,8 @@ export interface HandoffRow extends HandoffFacets {
|
|
|
35
35
|
trace?: TraceEntry[];
|
|
36
36
|
/** Each unit's arrival event, as its receiver reported it (dispatch on arrival). */
|
|
37
37
|
arrivals?: Record<string, Record<string, any>>;
|
|
38
|
+
/** Dispatch commands accepted as preparing; on arrival dispatch the row is received once each is ready. */
|
|
39
|
+
awaitingReady?: Set<string>;
|
|
38
40
|
/** Lifecycle item keys already on the row (lifecycle.ts). */
|
|
39
41
|
noted?: string[];
|
|
40
42
|
/** Parked with a limit on its items, which lifecycle notes must never use up. */
|
|
@@ -98,6 +100,10 @@ export declare class GatewayState {
|
|
|
98
100
|
holdDispatch: boolean;
|
|
99
101
|
/** Units at each machine that arrived and have not left: machine → unit → job. */
|
|
100
102
|
private held;
|
|
103
|
+
/** Jobs sent and taken, by machine: job → the row that sent it, kept while the machine holds the job's units. */
|
|
104
|
+
private sentJobs;
|
|
105
|
+
/** Jobs whose units left each machine since its last snapshot: they ended, a snapshot without them lost nothing. */
|
|
106
|
+
private endedJobs;
|
|
101
107
|
private openRows;
|
|
102
108
|
private closedRows;
|
|
103
109
|
readonly alarms: Map<string, AlarmTrack>;
|
|
@@ -122,6 +128,8 @@ export declare class GatewayState {
|
|
|
122
128
|
private strangers;
|
|
123
129
|
index(row: HandoffRow): void;
|
|
124
130
|
get(id: string): HandoffRow | undefined;
|
|
131
|
+
/** A row that ended, while the gateway still remembers it. */
|
|
132
|
+
closedRow(id: string): HandoffRow | undefined;
|
|
125
133
|
close(id: string, ended?: RowEnd): HandoffRow | undefined;
|
|
126
134
|
isClosed(id: string): boolean;
|
|
127
135
|
open(): HandoffRow[];
|
|
@@ -130,11 +138,26 @@ export declare class GatewayState {
|
|
|
130
138
|
closedFor(unitId: string, pred: (r: HandoffRow) => boolean): HandoffRow | undefined;
|
|
131
139
|
hold(machineId: string, unitId: string, jobId: string): void;
|
|
132
140
|
letGo(machineId: string, unitId: string): void;
|
|
141
|
+
/** A job the machine took, remembered by its row while the machine holds the job's units. */
|
|
142
|
+
rememberSent(machineId: string, jobId: string, rowId: string): void;
|
|
143
|
+
/**
|
|
144
|
+
* After a restart, the jobs the machine lost: those its units are held under
|
|
145
|
+
* or its last snapshot listed, that the new snapshot lacks and no open row
|
|
146
|
+
* will send again. `resend` names the rows remembered for them; `unknown`
|
|
147
|
+
* the jobs nothing is remembered for.
|
|
148
|
+
*/
|
|
149
|
+
lostAfterReboot(machineId: string, listed: Set<string>): {
|
|
150
|
+
resend: {
|
|
151
|
+
jobId: string;
|
|
152
|
+
row: HandoffRow;
|
|
153
|
+
}[];
|
|
154
|
+
unknown: string[];
|
|
155
|
+
};
|
|
133
156
|
/** The job a unit arrived under at `machineId`, while it is still there. */
|
|
134
157
|
heldJob(machineId: string, unitId: string): string | undefined;
|
|
135
158
|
heldAt(machineId: string): string[];
|
|
136
159
|
heldUnitsAt(machineId: string): HeldUnit[];
|
|
137
|
-
/** Keep only units whose job the machine still lists (its snapshot). */
|
|
160
|
+
/** Keep only units whose job the machine still lists (its snapshot); a job it no longer lists is over. */
|
|
138
161
|
keepHeld(machineId: string, jobIds: Set<string>): void;
|
|
139
162
|
/** Keep a sender's finished record until its row is known, so an arrival cannot close the row without it. */
|
|
140
163
|
rememberFinish(machineId: string, unitId: string, body: Record<string, any>): void;
|
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
*/
|
|
8
8
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
9
9
|
exports.GatewayState = exports.ROW_END = exports.DISPATCH_STATE = void 0;
|
|
10
|
+
const jobs_1 = require("./jobs");
|
|
10
11
|
const project_1 = require("./project");
|
|
11
12
|
const trace_1 = require("./trace");
|
|
12
13
|
/**
|
|
@@ -44,6 +45,10 @@ class GatewayState {
|
|
|
44
45
|
this.holdDispatch = false;
|
|
45
46
|
/** Units at each machine that arrived and have not left: machine → unit → job. */
|
|
46
47
|
this.held = new Map();
|
|
48
|
+
/** Jobs sent and taken, by machine: job → the row that sent it, kept while the machine holds the job's units. */
|
|
49
|
+
this.sentJobs = new Map();
|
|
50
|
+
/** Jobs whose units left each machine since its last snapshot: they ended, a snapshot without them lost nothing. */
|
|
51
|
+
this.endedJobs = new Map();
|
|
47
52
|
this.openRows = new Map();
|
|
48
53
|
this.closedRows = new Map();
|
|
49
54
|
this.alarms = new Map();
|
|
@@ -75,13 +80,17 @@ class GatewayState {
|
|
|
75
80
|
this.openRows.set(row.id, existing
|
|
76
81
|
? {
|
|
77
82
|
...row, dispatch: existing.dispatch, trace: (0, trace_1.mergeTrace)(row.trace, existing.trace), arrivals: existing.arrivals,
|
|
78
|
-
dueAt: existing.dueAt, noted: existing.noted ?? row.noted,
|
|
83
|
+
dueAt: existing.dueAt, noted: existing.noted ?? row.noted, awaitingReady: existing.awaitingReady,
|
|
79
84
|
}
|
|
80
85
|
: row);
|
|
81
86
|
}
|
|
82
87
|
get(id) {
|
|
83
88
|
return this.openRows.get(id);
|
|
84
89
|
}
|
|
90
|
+
/** A row that ended, while the gateway still remembers it. */
|
|
91
|
+
closedRow(id) {
|
|
92
|
+
return this.closedRows.get(id);
|
|
93
|
+
}
|
|
85
94
|
close(id, ended = exports.ROW_END.RESOLVED) {
|
|
86
95
|
const row = this.openRows.get(id);
|
|
87
96
|
if (!row)
|
|
@@ -115,7 +124,53 @@ class GatewayState {
|
|
|
115
124
|
this.held.get(machineId).set(unitId, jobId);
|
|
116
125
|
}
|
|
117
126
|
letGo(machineId, unitId) {
|
|
118
|
-
this.held.get(machineId)
|
|
127
|
+
const units = this.held.get(machineId);
|
|
128
|
+
const jobId = units?.get(unitId);
|
|
129
|
+
units?.delete(unitId);
|
|
130
|
+
if (jobId === undefined || [...units.values()].includes(jobId))
|
|
131
|
+
return;
|
|
132
|
+
this.sentJobs.get(machineId)?.delete(jobId);
|
|
133
|
+
if (!this.endedJobs.has(machineId))
|
|
134
|
+
this.endedJobs.set(machineId, new Set());
|
|
135
|
+
this.endedJobs.get(machineId).add(jobId);
|
|
136
|
+
}
|
|
137
|
+
/** A job the machine took, remembered by its row while the machine holds the job's units. */
|
|
138
|
+
rememberSent(machineId, jobId, rowId) {
|
|
139
|
+
if (!this.sentJobs.has(machineId))
|
|
140
|
+
this.sentJobs.set(machineId, new Map());
|
|
141
|
+
this.sentJobs.get(machineId).set(jobId, rowId);
|
|
142
|
+
}
|
|
143
|
+
/**
|
|
144
|
+
* After a restart, the jobs the machine lost: those its units are held under
|
|
145
|
+
* or its last snapshot listed, that the new snapshot lacks and no open row
|
|
146
|
+
* will send again. `resend` names the rows remembered for them; `unknown`
|
|
147
|
+
* the jobs nothing is remembered for.
|
|
148
|
+
*/
|
|
149
|
+
lostAfterReboot(machineId, listed) {
|
|
150
|
+
const link = this.links.get(machineId);
|
|
151
|
+
const before = new Set(this.heldAt(machineId));
|
|
152
|
+
for (const job of link?.jobs ?? [])
|
|
153
|
+
if (typeof job?.jobId === 'string')
|
|
154
|
+
before.add(job.jobId);
|
|
155
|
+
const ended = this.endedJobs.get(machineId) ?? new Set();
|
|
156
|
+
const sendsAgain = new Set();
|
|
157
|
+
for (const row of this.openRows.values()) {
|
|
158
|
+
if (row.toMachine === machineId)
|
|
159
|
+
for (const job of (0, jobs_1.dispatchJobs)(row.jobId, row.units, this.unitsPerDispatch))
|
|
160
|
+
sendsAgain.add(job.jobId);
|
|
161
|
+
}
|
|
162
|
+
const resend = [];
|
|
163
|
+
const unknown = [];
|
|
164
|
+
for (const jobId of before) {
|
|
165
|
+
if (listed.has(jobId) || ended.has(jobId) || sendsAgain.has(jobId))
|
|
166
|
+
continue;
|
|
167
|
+
const row = this.closedRows.get(this.sentJobs.get(machineId)?.get(jobId) ?? '');
|
|
168
|
+
if (row)
|
|
169
|
+
resend.push({ jobId, row });
|
|
170
|
+
else
|
|
171
|
+
unknown.push(jobId);
|
|
172
|
+
}
|
|
173
|
+
return { resend, unknown };
|
|
119
174
|
}
|
|
120
175
|
/** The job a unit arrived under at `machineId`, while it is still there. */
|
|
121
176
|
heldJob(machineId, unitId) {
|
|
@@ -127,14 +182,18 @@ class GatewayState {
|
|
|
127
182
|
heldUnitsAt(machineId) {
|
|
128
183
|
return [...(this.held.get(machineId)?.entries() ?? [])].map(([unitId, jobId]) => ({ unitId, jobId }));
|
|
129
184
|
}
|
|
130
|
-
/** Keep only units whose job the machine still lists (its snapshot). */
|
|
185
|
+
/** Keep only units whose job the machine still lists (its snapshot); a job it no longer lists is over. */
|
|
131
186
|
keepHeld(machineId, jobIds) {
|
|
187
|
+
this.endedJobs.delete(machineId);
|
|
132
188
|
const units = this.held.get(machineId);
|
|
133
189
|
if (!units)
|
|
134
190
|
return;
|
|
135
|
-
for (const [unitId, jobId] of units)
|
|
136
|
-
if (
|
|
137
|
-
|
|
191
|
+
for (const [unitId, jobId] of units) {
|
|
192
|
+
if (jobIds.has(jobId))
|
|
193
|
+
continue;
|
|
194
|
+
units.delete(unitId);
|
|
195
|
+
this.sentJobs.get(machineId)?.delete(jobId);
|
|
196
|
+
}
|
|
138
197
|
}
|
|
139
198
|
/** Keep a sender's finished record until its row is known, so an arrival cannot close the row without it. */
|
|
140
199
|
rememberFinish(machineId, unitId, body) {
|
|
@@ -268,6 +327,8 @@ class GatewayState {
|
|
|
268
327
|
retainedAlarms: this.retainedAlarms, readingRetained: this.readingRetained, takenAt: this.takenAt, progressStep: this.progressStep, holdDispatch: this.holdDispatch,
|
|
269
328
|
});
|
|
270
329
|
copyInto(this.held, copy.held, (units) => new Map(units));
|
|
330
|
+
copyInto(this.sentJobs, copy.sentJobs, (jobs) => new Map(jobs));
|
|
331
|
+
copyInto(this.endedJobs, copy.endedJobs, (jobs) => new Set(jobs));
|
|
271
332
|
copyInto(this.openRows, copy.openRows, (row) => ({ ...row, units: [...row.units], ...(row.noted ? { noted: [...row.noted] } : {}) }));
|
|
272
333
|
copyInto(this.closedRows, copy.closedRows, (row) => ({ ...row }));
|
|
273
334
|
copyInto(this.alarms, copy.alarms, (track) => ({ ...track, ...(track.noted ? { noted: [...track.noted] } : {}) }));
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The last wire events of a line, kept by its leader so a client reads the
|
|
3
|
+
* tape over HTTP instead of the event stream. A ring of a fixed size; reads
|
|
4
|
+
* are newest first and may narrow by machine, direction, kind and time.
|
|
5
|
+
*/
|
|
6
|
+
import type { WireDirection, WireEvent, WireKind, WireWindowQuery } from '../../../types/machines-view';
|
|
7
|
+
export declare const DEFAULT_WIRE_WINDOW = 500;
|
|
8
|
+
export declare const MAX_WIRE_PAGE = 500;
|
|
9
|
+
export declare class WireWindow {
|
|
10
|
+
readonly size: number;
|
|
11
|
+
private readonly events;
|
|
12
|
+
private head;
|
|
13
|
+
private count;
|
|
14
|
+
constructor(size?: number);
|
|
15
|
+
push(event: WireEvent): void;
|
|
16
|
+
/** Newest first, narrowed by the query, at most `limit` (the window's size when absent). */
|
|
17
|
+
read(query?: WireWindowQuery): WireEvent[];
|
|
18
|
+
get length(): number;
|
|
19
|
+
}
|
|
20
|
+
export declare function isWireDirection(v: unknown): v is WireDirection;
|
|
21
|
+
export declare function isWireKind(v: unknown): v is WireKind;
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* The last wire events of a line, kept by its leader so a client reads the
|
|
4
|
+
* tape over HTTP instead of the event stream. A ring of a fixed size; reads
|
|
5
|
+
* are newest first and may narrow by machine, direction, kind and time.
|
|
6
|
+
*/
|
|
7
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
8
|
+
exports.WireWindow = exports.MAX_WIRE_PAGE = exports.DEFAULT_WIRE_WINDOW = void 0;
|
|
9
|
+
exports.isWireDirection = isWireDirection;
|
|
10
|
+
exports.isWireKind = isWireKind;
|
|
11
|
+
exports.DEFAULT_WIRE_WINDOW = 500;
|
|
12
|
+
exports.MAX_WIRE_PAGE = 500;
|
|
13
|
+
class WireWindow {
|
|
14
|
+
constructor(size = exports.DEFAULT_WIRE_WINDOW) {
|
|
15
|
+
this.size = size;
|
|
16
|
+
this.events = [];
|
|
17
|
+
this.head = 0;
|
|
18
|
+
this.count = 0;
|
|
19
|
+
if (!Number.isInteger(size) || size < 1)
|
|
20
|
+
throw new Error(`a wire window keeps at least one event (got ${size})`);
|
|
21
|
+
}
|
|
22
|
+
push(event) {
|
|
23
|
+
this.events[this.head] = event;
|
|
24
|
+
this.head = (this.head + 1) % this.size;
|
|
25
|
+
if (this.count < this.size)
|
|
26
|
+
this.count += 1;
|
|
27
|
+
}
|
|
28
|
+
/** Newest first, narrowed by the query, at most `limit` (the window's size when absent). */
|
|
29
|
+
read(query = {}) {
|
|
30
|
+
const limit = Math.min(Math.max(1, query.limit ?? this.size), exports.MAX_WIRE_PAGE);
|
|
31
|
+
const out = [];
|
|
32
|
+
for (let i = 1; i <= this.count && out.length < limit; i++) {
|
|
33
|
+
const event = this.events[(this.head - i + this.size) % this.size];
|
|
34
|
+
if (matches(event, query))
|
|
35
|
+
out.push(event);
|
|
36
|
+
}
|
|
37
|
+
return out;
|
|
38
|
+
}
|
|
39
|
+
get length() {
|
|
40
|
+
return this.count;
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
exports.WireWindow = WireWindow;
|
|
44
|
+
function matches(event, q) {
|
|
45
|
+
if (q.machine && event.machineId !== q.machine)
|
|
46
|
+
return false;
|
|
47
|
+
if (q.direction && event.direction !== q.direction)
|
|
48
|
+
return false;
|
|
49
|
+
if (q.kind && event.kind !== q.kind)
|
|
50
|
+
return false;
|
|
51
|
+
if (q.since && event.at <= q.since)
|
|
52
|
+
return false;
|
|
53
|
+
return true;
|
|
54
|
+
}
|
|
55
|
+
function isWireDirection(v) {
|
|
56
|
+
return v === 'in' || v === 'out';
|
|
57
|
+
}
|
|
58
|
+
const KINDS = ['presence', 'snapshot', 'job', 'alarm', 'telemetry', 'report', 'cmd', 'reply', 'receipt', 'unknown'];
|
|
59
|
+
function isWireKind(v) {
|
|
60
|
+
return typeof v === 'string' && KINDS.includes(v);
|
|
61
|
+
}
|
|
@@ -165,6 +165,7 @@ function plannedAction(a) {
|
|
|
165
165
|
case 'watchOffline': return { verb: 'none', detail: `watching ${a.machineId} for an offline row` };
|
|
166
166
|
case 'settleReply': return { verb: 'ack', detail: `${a.status} ${a.cmdId}` };
|
|
167
167
|
case 'settleSnapshot': return { verb: 'ack', detail: `${a.machineId} holds ${count(a.jobIds.length, 'job')}` };
|
|
168
|
+
case 'unrecoverable': return { verb: 'none', detail: `${a.machineId} lost ${a.jobs.join(', ')}; nothing remembered to send again` };
|
|
168
169
|
case 'bindAddress': return { verb: 'ack', detail: `bind ${a.machineId} to ${a.address}` };
|
|
169
170
|
case 'refuseBinding': return { verb: 'refuse', detail: a.reason };
|
|
170
171
|
}
|
|
@@ -30,6 +30,8 @@ export declare class LineLeadership {
|
|
|
30
30
|
private leadOnce;
|
|
31
31
|
/** Lead while the lock is held; the error that ended it. */
|
|
32
32
|
private holdLock;
|
|
33
|
+
/** The leader gives its turn up without stopping: `lost` runs, the lock goes, and the line contends again after the pause. */
|
|
34
|
+
relinquish(reason: string): void;
|
|
33
35
|
/** A leader unlocks; a replica still waiting drops its connection, which ends the wait. */
|
|
34
36
|
stop(): Promise<void>;
|
|
35
37
|
}
|
|
@@ -62,7 +62,7 @@ class LineLeadership {
|
|
|
62
62
|
async holdLock(client, lead) {
|
|
63
63
|
const ended = new Promise((resolve) => {
|
|
64
64
|
client.once('error', resolve);
|
|
65
|
-
this.endTurn = () => resolve(new Error('leadership stopped'));
|
|
65
|
+
this.endTurn = (err) => resolve(err ?? new Error('leadership stopped'));
|
|
66
66
|
});
|
|
67
67
|
try {
|
|
68
68
|
await client.query(LOCK_SQL, [this.key]);
|
|
@@ -76,6 +76,12 @@ class LineLeadership {
|
|
|
76
76
|
return err;
|
|
77
77
|
}
|
|
78
78
|
}
|
|
79
|
+
/** The leader gives its turn up without stopping: `lost` runs, the lock goes, and the line contends again after the pause. */
|
|
80
|
+
relinquish(reason) {
|
|
81
|
+
if (!this.leading)
|
|
82
|
+
return;
|
|
83
|
+
this.endTurn?.(new Error(reason));
|
|
84
|
+
}
|
|
79
85
|
/** A leader unlocks; a replica still waiting drops its connection, which ends the wait. */
|
|
80
86
|
async stop() {
|
|
81
87
|
this.stopped = true;
|
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* The roles a line needs, as code-owned role declarations: one station role
|
|
3
3
|
* per process (a machine resolves the handoffs into it), the exit role (a
|
|
4
|
-
* person receives finished units)
|
|
5
|
-
* failed handoffs and closes alarm and machine rows)
|
|
6
|
-
*
|
|
7
|
-
*
|
|
4
|
+
* person receives finished units), the operator role (a person decides
|
|
5
|
+
* failed handoffs and closes alarm and machine rows) and the quality role
|
|
6
|
+
* (a person decides units finished on hold). Each role owns its form; the
|
|
7
|
+
* operator and quality roles enforce it on the server, since handoff rows
|
|
8
|
+
* also collect machine records the gateway already validates.
|
|
8
9
|
*/
|
|
9
10
|
import type { LTRoleConfig } from '../../types/startup';
|
|
10
11
|
import { type LineDef } from '../../lib/machines/line';
|
|
@@ -2,10 +2,11 @@
|
|
|
2
2
|
/**
|
|
3
3
|
* The roles a line needs, as code-owned role declarations: one station role
|
|
4
4
|
* per process (a machine resolves the handoffs into it), the exit role (a
|
|
5
|
-
* person receives finished units)
|
|
6
|
-
* failed handoffs and closes alarm and machine rows)
|
|
7
|
-
*
|
|
8
|
-
*
|
|
5
|
+
* person receives finished units), the operator role (a person decides
|
|
6
|
+
* failed handoffs and closes alarm and machine rows) and the quality role
|
|
7
|
+
* (a person decides units finished on hold). Each role owns its form; the
|
|
8
|
+
* operator and quality roles enforce it on the server, since handoff rows
|
|
9
|
+
* also collect machine records the gateway already validates.
|
|
9
10
|
*/
|
|
10
11
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
11
12
|
exports.machineRoleConfigs = machineRoleConfigs;
|
|
@@ -44,6 +45,14 @@ function machineRoleConfigs(line) {
|
|
|
44
45
|
enforce_schema: true,
|
|
45
46
|
...BOARD,
|
|
46
47
|
},
|
|
48
|
+
{
|
|
49
|
+
role: (0, line_1.qualityRole)(line),
|
|
50
|
+
title: 'Line quality',
|
|
51
|
+
description: 'Units a machine finished on hold: release, rework or scrap.',
|
|
52
|
+
form_schema: forms_1.QUALITY_FORM_SCHEMA,
|
|
53
|
+
enforce_schema: true,
|
|
54
|
+
...BOARD,
|
|
55
|
+
},
|
|
47
56
|
];
|
|
48
57
|
}
|
|
49
58
|
/** The host's declarations first; line roles only where the host declared none. */
|
|
@@ -16,9 +16,10 @@ export type LineMessaging = Pick<typeof messagingRegistry, 'streams' | 'pubsub'>
|
|
|
16
16
|
/**
|
|
17
17
|
* The gateway's view of every machine on the line. With `nats`, events are read
|
|
18
18
|
* from Long Tail's messaging streams; MACHINE_SITE_NATS_URL instead names a NATS
|
|
19
|
-
* server of the site's own, which the bus connects to itself.
|
|
19
|
+
* server of the site's own, which the bus connects to itself. `siteCodecs` are
|
|
20
|
+
* the codecs of every line on the site, so the streams keep all their subjects.
|
|
20
21
|
*/
|
|
21
|
-
export declare function createMachineBus(config: MachineLineConfig, line: LineDef, codec?: LineCodec, messaging?: LineMessaging): MachineBus;
|
|
22
|
+
export declare function createMachineBus(config: MachineLineConfig, line: LineDef, codec?: LineCodec, messaging?: LineMessaging, siteCodecs?: LineCodec[], onLost?: (reason: string) => void): MachineBus;
|
|
22
23
|
/** Long Tail's escalation events for the gateway; `in-process` needs the running instance's callback adapter. */
|
|
23
24
|
export declare function createEscalationFeed(config: MachineLineConfig, line: LineDef, callbacks?: CallbackEventAdapter): EscalationFeed;
|
|
24
25
|
/** One MQTT link per simulated machine; machines speak MQTT in every networked mode. */
|
|
@@ -33,12 +33,13 @@ function defaultCodec(config, line) {
|
|
|
33
33
|
/**
|
|
34
34
|
* The gateway's view of every machine on the line. With `nats`, events are read
|
|
35
35
|
* from Long Tail's messaging streams; MACHINE_SITE_NATS_URL instead names a NATS
|
|
36
|
-
* server of the site's own, which the bus connects to itself.
|
|
36
|
+
* server of the site's own, which the bus connects to itself. `siteCodecs` are
|
|
37
|
+
* the codecs of every line on the site, so the streams keep all their subjects.
|
|
37
38
|
*/
|
|
38
|
-
function createMachineBus(config, line, codec = defaultCodec(config, line), messaging = messaging_1.messagingRegistry) {
|
|
39
|
+
function createMachineBus(config, line, codec = defaultCodec(config, line), messaging = messaging_1.messagingRegistry, siteCodecs = [], onLost) {
|
|
39
40
|
switch (config.transport) {
|
|
40
41
|
case config_1.MACHINE_TRANSPORT.NATS: {
|
|
41
|
-
const where = { mqtt: mqttOf(config), controllerId: (0, line_1.controllerId)(line), siteId: line.siteId, lineId: line.lineId, codec };
|
|
42
|
+
const where = { mqtt: mqttOf(config), controllerId: (0, line_1.controllerId)(line), siteId: line.siteId, lineId: line.lineId, codec, siteCodecs, onLost };
|
|
42
43
|
if (config.siteNatsUrl) {
|
|
43
44
|
return new nats_bus_1.NatsMachineBus({ natsUrl: config.siteNatsUrl, user: config.siteNatsUser, password: config.siteNatsPassword, ...where });
|
|
44
45
|
}
|
|
@@ -11,7 +11,7 @@ import type { CallbackEventAdapter } from '../../lib/events/callback';
|
|
|
11
11
|
import { type LineDef } from '../../lib/machines/line';
|
|
12
12
|
import { type LineCodec } from '../../lib/machines/protocol';
|
|
13
13
|
import type { InboundMessage } from '../../lib/machines/transport';
|
|
14
|
-
import type { InspectResult, LineSummary, LineView } from '../../types/machines-view';
|
|
14
|
+
import type { InspectResult, LineSummary, LineView, WireEvent, WireWindowQuery } from '../../types/machines-view';
|
|
15
15
|
import { type MachineLineConfig } from './config';
|
|
16
16
|
import { type GatewayLog } from './gateway/log';
|
|
17
17
|
import type { HandoffRowAdapter } from './rows';
|
|
@@ -23,8 +23,12 @@ export interface MachineSurfaceOptions {
|
|
|
23
23
|
callbacks?: CallbackEventAdapter;
|
|
24
24
|
log?: GatewayLog;
|
|
25
25
|
codec?: LineCodec;
|
|
26
|
+
/** A codec for one line, keyed `<siteId>/<lineId>`; every other line uses `codec`. */
|
|
27
|
+
codecs?: Record<string, LineCodec>;
|
|
26
28
|
rows?: HandoffRowAdapter;
|
|
27
29
|
}
|
|
30
|
+
/** The codec a line runs on: its own, the surface's, or the generic line protocol. */
|
|
31
|
+
export declare function codecFor(opts: Pick<MachineSurfaceOptions, 'codec' | 'codecs'>, config: MachineLineConfig, line: LineDef): LineCodec;
|
|
28
32
|
/** A message to dry-run against a line. */
|
|
29
33
|
export type WireProbe = Pick<InboundMessage, 'topic' | 'payload' | 'retained'>;
|
|
30
34
|
/** What settings and the API read of the surface. */
|
|
@@ -35,6 +39,8 @@ export interface LineSurface {
|
|
|
35
39
|
describeLine(siteId: string, lineId: string): LineView | null;
|
|
36
40
|
/** A dry run of one message; null when this replica is not leading the line. */
|
|
37
41
|
inspect(siteId: string, lineId: string, probe: WireProbe): InspectResult | null;
|
|
42
|
+
/** The leader's last wire events, newest first; null when this replica is not leading the line. */
|
|
43
|
+
wire(siteId: string, lineId: string, query?: WireWindowQuery): WireEvent[] | null;
|
|
38
44
|
/** The declared line, or undefined. */
|
|
39
45
|
findLine(siteId: string, lineId: string): LineDef | undefined;
|
|
40
46
|
/** The ops a machine-command row on this line may ask for. */
|
|
@@ -65,8 +71,11 @@ export declare class MachineSurface implements LineSurface {
|
|
|
65
71
|
describeLines(): LineSummary[];
|
|
66
72
|
describeLine(siteId: string, lineId: string): LineView | null;
|
|
67
73
|
inspect(siteId: string, lineId: string, probe: WireProbe): InspectResult | null;
|
|
74
|
+
wire(siteId: string, lineId: string, query?: WireWindowQuery): WireEvent[] | null;
|
|
68
75
|
findLine(siteId: string, lineId: string): LineDef | undefined;
|
|
69
76
|
commandOps(siteId: string, lineId: string): string[];
|
|
77
|
+
/** The codecs every line on a site runs, so one site's streams keep all their subjects. */
|
|
78
|
+
codecsOnSite(siteId: string): LineCodec[];
|
|
70
79
|
private begin;
|
|
71
80
|
private run;
|
|
72
81
|
}
|