@nylorun/runtime 0.21.1-beta → 0.23.0-beta
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +32 -0
- package/README.md +17 -3
- package/dist/adapters/execution/restate.d.ts +15 -3
- package/dist/adapters/execution/restate.js +77 -83
- package/dist/adapters/sandbox/local.js +1 -1
- package/dist/adapters/sandbox/virtual.js +1 -1
- package/dist/adapters/streams/s2.js +1 -13
- package/dist/api/components.d.ts +42 -2
- package/dist/core/flow-host.d.ts +1 -19
- package/dist/core/flow-host.js +8 -37
- package/dist/execution/types.d.ts +10 -8
- package/dist/execution/types.js +5 -7
- package/dist/gates/egress-credentials.d.ts +48 -0
- package/dist/gates/egress-credentials.js +147 -0
- package/dist/gates/egress.d.ts +3 -0
- package/dist/gates/egress.js +14 -1
- package/dist/gates/tenant-vaults.d.ts +4 -0
- package/dist/gates/tenant-vaults.js +2 -0
- package/dist/harness/calls.d.ts +2 -0
- package/dist/harness/calls.js +1 -0
- package/dist/harness/executors.js +1 -0
- package/dist/harness/main.js +4 -1
- package/dist/harness/pod.d.ts +2 -0
- package/dist/harness/pod.js +36 -1
- package/dist/harness-api/start.d.ts +2 -0
- package/dist/harness-api/start.js +4 -2
- package/dist/harness-api/ws-server.js +2 -2
- package/dist/host/execution.d.ts +8 -6
- package/dist/host/gates.js +11 -0
- package/dist/host/main.js +10 -16
- package/dist/host/stack-config.d.ts +6 -0
- package/dist/host/stack-config.js +6 -0
- package/dist/infra/execution.d.ts +18 -1
- package/dist/infra/execution.js +28 -4
- package/dist/infra/index.d.ts +1 -1
- package/dist/keys/egress-ca.d.ts +15 -0
- package/dist/keys/egress-ca.js +66 -0
- package/dist/keys/keys.d.ts +8 -1
- package/dist/keys/keys.js +6 -0
- package/dist/keys/x509.d.ts +29 -0
- package/dist/keys/x509.js +160 -0
- package/dist/management-openapi.json +195 -9
- package/dist/openapi.json +2 -1
- package/dist/retry.d.ts +21 -0
- package/dist/retry.js +34 -0
- package/dist/sandbox/join.d.ts +2 -0
- package/dist/sandbox/join.js +14 -10
- package/dist/sandbox/manager.d.ts +2 -0
- package/dist/sandbox/manager.js +1 -1
- package/dist/sandbox/pods/reconcile.js +3 -3
- package/dist/sandbox/tools.d.ts +3 -1
- package/dist/sandbox/tools.js +4 -2
- package/dist/sandbox/types.d.ts +5 -0
- package/dist/store/postgres/control.js +8 -1
- package/dist/store/postgres/drizzle/0018_wake_outbox_retries.sql +3 -0
- package/dist/store/postgres/drizzle/0019_sandbox_signal_outbox.sql +18 -0
- package/dist/store/postgres/drizzle/meta/_journal.json +14 -0
- package/dist/store/postgres/schema.d.ts +265 -4
- package/dist/store/postgres/schema.js +35 -4
- package/dist/store/postgres/store.js +165 -52
- package/dist/store/types.d.ts +96 -39
- package/dist/streams/relay/core.js +1 -13
- package/dist/tenant/advance.js +6 -7
- package/dist/tenant/commands.js +1 -2
- package/dist/tenant/context.d.ts +19 -11
- package/dist/tenant/control.d.ts +8 -1
- package/dist/tenant/control.js +2 -0
- package/dist/tenant/runtime.js +23 -3
- package/dist/tenant/sandboxes.d.ts +2 -1
- package/dist/tenant/sandboxes.js +23 -19
- package/dist/tenant/scheduler.d.ts +7 -0
- package/dist/tenant/scheduler.js +38 -7
- package/dist/tenant/session-streams.d.ts +0 -1
- package/dist/tenant/session-streams.js +1 -13
- package/dist/tenant/sessions.js +26 -0
- package/dist/tenant/streams.js +14 -29
- package/dist/tenant/sweep.d.ts +39 -12
- package/dist/tenant/sweep.js +157 -58
- package/dist/tenant/transcript.d.ts +8 -8
- package/dist/vault/service.d.ts +34 -0
- package/dist/vault/service.js +148 -15
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/package.json +4 -3
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,37 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.23.0-beta
|
|
4
|
+
|
|
5
|
+
### Minor Changes
|
|
6
|
+
|
|
7
|
+
- 72a4b52: The outbox no longer stalls the Tenant sweep, holds up commands or loses a pod sandbox's signals, and Workers register versioned Restate deployments.
|
|
8
|
+
|
|
9
|
+
- A wake that keeps failing (Restate refuses it, or an older Runtime does not know its reason) is retried with backoff and, after ten failures, parked: logged once at error and kept in the outbox for an operator. It no longer fails the sweep, so Restate never pauses the Tenant's sweep object over it. Migration 0018 adds `attempts`, `retry_at` and `parked_at` to `nylorun.wakes`.
|
|
10
|
+
- The sweep drains the whole outbox and pages through every orphaned session in each pass, within a 2 s budget, instead of 100 rows per pass.
|
|
11
|
+
- A command's wake is sent after commit without the request waiting for it; a failed send is logged at warn and sent again by the sweep's next pass. The sweep's grace is compared on the database's clock and is now 30 s.
|
|
12
|
+
- A pod sandbox's reconcile and idle timer go through the outbox (migration 0019, `nylorun.sandbox_signals`), so a crash after commit no longer loses them, and a host revocation goes over the control bus (`host.revoked`), so it reaches every process with the Tenant open. `Tx.afterCommit` is removed.
|
|
13
|
+
- Sends to Restate's ingress use `@restatedev/restate-sdk-clients` 1.17.2, each with an idempotency key.
|
|
14
|
+
- A Worker registers `<NYLORUN_WORKER_URL>/nylorun/<version>`, the version being the new `NYLORUN_WORKER_VERSION` or the Runtime's own, without forcing in a container, so a rolling upgrade keeps the old deployment for the invocations it started. A build whose Worker code changed under the same Runtime version sets its own `NYLORUN_WORKER_VERSION`.
|
|
15
|
+
|
|
16
|
+
## 0.22.0-beta
|
|
17
|
+
|
|
18
|
+
### Minor Changes
|
|
19
|
+
|
|
20
|
+
- 212364b: **Credentials for skills (R2c).** A skill's CLI in a pod sandbox authenticates with a vault credential whose value never enters the sandbox.
|
|
21
|
+
|
|
22
|
+
- `@nylorun/core`: vault credentials gain `environment_secret` (`secretName`, `secretValue`, `allowedHosts`, optional `inject: { header, format }`) and `environment_variable` (`variableName`, `variableValue`), in create, rotate and `CredentialInfo` (whose `binding.url` is now optional). `ENVIRONMENT_SECRET_SENTINEL` (`nylorun-managed`), `ENVIRONMENT_SECRET_DEFAULT_INJECT`, `isInjectFormat`, `renderInjectFormat`, `EnvironmentNameSchema` and `RESERVED_ENVIRONMENT_NAMES` are exported; `ERROR_CODES` gains `credential_conflict`; the Harness API's `RunRouting.sandbox` gains `environment`.
|
|
23
|
+
- `@nylorun/runtime`: every sandbox command gets each secret's name set to `nylorun-managed` and each variable's value. egress-gate terminates TLS for port 443 of a host an `environment_secret` of the sandbox's sessions is bound to, with a fresh leaf signed by the installation's egress CA (held by keys; its certificate reaches the pod in the join answer, and the engine points `SSL_CERT_FILE`, `NODE_EXTRA_CA_CERTS`, `REQUESTS_CA_BUNDLE`, `CURL_CA_BUNDLE`, `GIT_SSL_CAINFO` and `AWS_CA_BUNDLE` at a bundle with it), and sets the credential's header on every request from the vault, uncached. Every other host stays an opaque tunnel. Session open answers `409 credential_conflict` for two credentials with one variable name or one host, or two owners' secrets on one sandbox. No protocol change.
|
|
24
|
+
- `@nylorun/studio`: the Credentials page lists shell credentials (their variable and hosts) and deletes them; create and rotate them with the Management API.
|
|
25
|
+
|
|
26
|
+
### Patch Changes
|
|
27
|
+
|
|
28
|
+
- Pin core to the tested release.
|
|
29
|
+
- Pin harness to the tested release.
|
|
30
|
+
- Updated dependencies [212364b]
|
|
31
|
+
- Updated dependencies
|
|
32
|
+
- @nylorun/core@0.19.0-beta
|
|
33
|
+
- @nylorun/harness@0.25.1-beta
|
|
34
|
+
|
|
3
35
|
## 0.21.1-beta
|
|
4
36
|
|
|
5
37
|
### Patch Changes
|
package/README.md
CHANGED
|
@@ -44,8 +44,9 @@ or deleted key stops authenticating at once. A database written by a Runtime tha
|
|
|
44
44
|
database (`tenant_<id>` schemas), or by a pre-release build of one Tenant per database
|
|
45
45
|
(`schema_version` tables), is refused: this release starts fresh on a new database. Restate runs one advance of a session at a time, delivers its wakes and holds
|
|
46
46
|
its timers and the Tenant's sweep (Durable Session Execution), and journals no effect:
|
|
47
|
-
checkpoints, effects, events and the
|
|
48
|
-
written to the outbox in the transaction that causes it, so a commit
|
|
47
|
+
checkpoints, effects, events and the outbox are the Session Store's, and a wake or a pod
|
|
48
|
+
sandbox's reconcile is written to the outbox in the transaction that causes it, so a commit
|
|
49
|
+
never loses it ([src/CONTEXT.md](src/CONTEXT.md)); every session's events are relayed from the record to
|
|
49
50
|
its own S2 stream, which history and SSE read (Durable Streams). One image runs every **service**,
|
|
50
51
|
and `--service` picks what a process runs: `core` serves the Runtime API, the
|
|
51
52
|
Management API and SSE and runs the stream relay, `loop` runs advances (the Worker), and
|
|
@@ -69,10 +70,23 @@ gateway ([DEPLOYMENT.md](../DEPLOYMENT.md#the-harness-agent-turns-mcp-servers-an
|
|
|
69
70
|
|
|
70
71
|
The container is configured by its environment, which a local Tenant's Compose file
|
|
71
72
|
sets: `NYLORUN_DATABASE_URL` (required), `NYLORUN_RESTATE_INGRESS_URL`,
|
|
72
|
-
`NYLORUN_RESTATE_ADMIN_URL`, `NYLORUN_WORKER_URL` and
|
|
73
|
+
`NYLORUN_RESTATE_ADMIN_URL`, `NYLORUN_WORKER_URL`, `NYLORUN_WORKER_VERSION` and
|
|
73
74
|
`NYLORUN_RESTATE_IDENTITY_KEY` (Restate), `NYLORUN_S2_ENDPOINT` and
|
|
74
75
|
`NYLORUN_S2_TOKEN`, and in container mode `NYLORUN_LISTEN_HOST`,
|
|
75
76
|
`NYLORUN_LISTEN_PORT`, `NYLORUN_ALLOWED_HOSTS` and `NYLORUN_PUBLIC_URL`.
|
|
77
|
+
A Worker registers its Restate deployment at a versioned URL,
|
|
78
|
+
`<NYLORUN_WORKER_URL>/nylorun/<version>`, where the version is `NYLORUN_WORKER_VERSION`
|
|
79
|
+
(letters, digits, `.`, `_`, `-`) or, by default, the Runtime's own. In a container it never
|
|
80
|
+
replaces a deployment already registered there (`force: false`): during a rolling upgrade
|
|
81
|
+
the new version's Workers register next to the old ones, Restate sends new invocations to
|
|
82
|
+
the new deployment and lets the old one finish what it started, and every Worker of one
|
|
83
|
+
version shares its deployment through one load-balanced `NYLORUN_WORKER_URL`. A build that
|
|
84
|
+
changes the Worker's code without changing the Runtime's version (an image built from a
|
|
85
|
+
branch) sets its own `NYLORUN_WORKER_VERSION`: without force Restate keeps the deployment it
|
|
86
|
+
already has at a URL and does not discover the Worker again, so the changed code would run
|
|
87
|
+
under the old deployment's registration, and a Worker whose services that deployment lacks
|
|
88
|
+
refuses to start. Outside a container (a development
|
|
89
|
+
Host) the registration replaces the deployment at that URL.
|
|
76
90
|
`NYLORUN_MANAGEMENT_KEY_FILE` names the bootstrap secret (below).
|
|
77
91
|
`NYLORUN_IDENTITY_FILE` names the identity
|
|
78
92
|
file, a YAML list of the trusted issuers whose JWTs the Runtime API accepts
|
|
@@ -1,4 +1,10 @@
|
|
|
1
1
|
import { type DurableExecution, type SandboxSignal, type StuckInvocation, type Wake, type WorkerHandlers } from "../../execution/types.js";
|
|
2
|
+
/**
|
|
3
|
+
* How long one send through the ingress keeps retrying while the ingress is unreachable,
|
|
4
|
+
* overloaded, or answers 404 because a just-registered service has not reached it yet. The
|
|
5
|
+
* outbox's grace (`OUTBOX_GRACE_MS` in `tenant/sweep.ts`) is longer.
|
|
6
|
+
*/
|
|
7
|
+
export declare const SEND_RETRY_MS = 10000;
|
|
2
8
|
export interface RestateTimeouts {
|
|
3
9
|
/**
|
|
4
10
|
* How long an invocation may run without journal progress before Restate
|
|
@@ -60,8 +66,8 @@ export interface RestateExecutionOptions {
|
|
|
60
66
|
/**
|
|
61
67
|
* Overwrite a deployment already registered at `workerAdvertisedUrl`, so a
|
|
62
68
|
* restarted Worker with changed code takes effect. Default true, which suits
|
|
63
|
-
* a stable URL on a developer machine.
|
|
64
|
-
* versioned URL
|
|
69
|
+
* a stable URL on a developer machine. A Host in a container advertises a
|
|
70
|
+
* versioned URL without it (`workerDeployment` in `infra/execution.ts`, §14.6).
|
|
65
71
|
*/
|
|
66
72
|
forceRegistration?: boolean;
|
|
67
73
|
/** How long `start` keeps retrying registration while Restate comes up. Default 60000. */
|
|
@@ -88,6 +94,7 @@ export declare class RestateExecution implements DurableExecution {
|
|
|
88
94
|
private readonly inflight;
|
|
89
95
|
private readonly names;
|
|
90
96
|
private readonly ingressUrl;
|
|
97
|
+
private readonly ingress;
|
|
91
98
|
private readonly adminUrl;
|
|
92
99
|
private readonly sweepIntervalMs;
|
|
93
100
|
constructor(options: RestateExecutionOptions);
|
|
@@ -123,7 +130,12 @@ export declare class RestateExecution implements DurableExecution {
|
|
|
123
130
|
private runTracked;
|
|
124
131
|
private staleSweepMs;
|
|
125
132
|
private serviceOptions;
|
|
126
|
-
/**
|
|
133
|
+
/**
|
|
134
|
+
* One-way send through the ingress (`@restatedev/restate-sdk-clients`). Every send carries
|
|
135
|
+
* an idempotency key, so the client's retries (`SEND_RETRY_MS`), and a send repeated after
|
|
136
|
+
* its answer was lost, start one invocation: the request's own key where it has one (an
|
|
137
|
+
* outbox row's, a timer's time), or one made for this call.
|
|
138
|
+
*/
|
|
127
139
|
private send;
|
|
128
140
|
/** Registers the Worker endpoint with the admin API, retrying until Restate is up. */
|
|
129
141
|
private register;
|
|
@@ -25,37 +25,28 @@
|
|
|
25
25
|
* What a run answers arms its timers and its retry.
|
|
26
26
|
*
|
|
27
27
|
* Wakes, sandbox reconciles, timers and sweep arming go through the ingress as one-way
|
|
28
|
-
* sends, so an API node can call them without ever
|
|
29
|
-
*
|
|
30
|
-
*
|
|
28
|
+
* sends (`@restatedev/restate-sdk-clients`), so an API node can call them without ever
|
|
29
|
+
* calling `start`. Every send carries an idempotency key, which the client's retries need: a
|
|
30
|
+
* wake's dedupe key (the outbox gives every committed wake one), a reconcile's outbox key, a
|
|
31
|
+
* timer's `<key>@<at>`, a sandbox timer's `<timer>@<at>`, or one made for the call. A send
|
|
32
|
+
* repeated after its answer was lost therefore starts no second invocation.
|
|
31
33
|
*
|
|
32
|
-
*
|
|
33
|
-
*
|
|
34
|
-
* Restate owns when a session runs: it delivers each wake at least once, deduped by its
|
|
35
|
-
* key for `dedupeRetentionMs`; runs at most one invocation per key at a time; retries an
|
|
36
|
-
* invocation whose attempt failed, so an advance whose Worker died runs again and takes the
|
|
37
|
-
* session over (§11.4); and keeps every timer: the busy re-wake, `NylorunTimer`,
|
|
38
|
-
* `NylorunSandbox` and the `NylorunTenant` sweep chain. The Session Store (Postgres) owns
|
|
39
|
-
* what happened: the checkpoint, the effect journal (`harness-api/record.ts`), the events
|
|
40
|
-
* (the Record), and the wake outbox that hands each committed wake to Restate. Restate
|
|
41
|
-
* journals no effect: an effect's intent and outcome commit with the session's events in
|
|
42
|
-
* one transaction, under the advance's lease, and remote harnesses and pod engines write
|
|
43
|
-
* them too, over the Harness API.
|
|
44
|
-
*
|
|
45
|
-
* Two Session Store mechanisms overlap Restate on purpose:
|
|
46
|
-
*
|
|
47
|
-
* - **The lease and its epoch** (§10.6). Restate's one invocation per key holds for an
|
|
48
|
-
* attempt, not for the code it started: an attempt that ends (abort timeout, a dropped
|
|
49
|
-
* connection) is retried while the previous attempt's handler may still run on its Worker,
|
|
50
|
-
* and a remote harness or pod engine may still hold the run. The epoch fences their writes,
|
|
51
|
-
* and `busy` answers the overlap.
|
|
52
|
-
* - **The advance deadline** (`tenant/worker.ts`). Restate's abort timeout does not stop a
|
|
53
|
-
* running handler, so the Worker bounds each advance itself, below `timeouts`.
|
|
34
|
+
* What Restate owns and what the Session Store owns, and where the two overlap on purpose,
|
|
35
|
+
* is said once, under "Durable Session Execution" in `src/CONTEXT.md`.
|
|
54
36
|
*/
|
|
37
|
+
import { randomUUID } from "node:crypto";
|
|
55
38
|
import { createServer } from "node:http2";
|
|
56
39
|
import * as restate from "@restatedev/restate-sdk";
|
|
40
|
+
import * as clients from "@restatedev/restate-sdk-clients";
|
|
41
|
+
import { retry } from "../../retry.js";
|
|
57
42
|
import { DEFAULT_STOP_GRACE_MS, WAKE_REASONS, parseSessionKey, sessionKey, settleWithin, } from "../../execution/types.js";
|
|
58
43
|
const HOUR_MS = 60 * 60 * 1000;
|
|
44
|
+
/**
|
|
45
|
+
* How long one send through the ingress keeps retrying while the ingress is unreachable,
|
|
46
|
+
* overloaded, or answers 404 because a just-registered service has not reached it yet. The
|
|
47
|
+
* outbox's grace (`OUTBOX_GRACE_MS` in `tenant/sweep.ts`) is longer.
|
|
48
|
+
*/
|
|
49
|
+
export const SEND_RETRY_MS = 10_000;
|
|
59
50
|
export function createRestateExecution(options) {
|
|
60
51
|
return new RestateExecution(options);
|
|
61
52
|
}
|
|
@@ -69,6 +60,7 @@ export class RestateExecution {
|
|
|
69
60
|
inflight = new Set();
|
|
70
61
|
names;
|
|
71
62
|
ingressUrl;
|
|
63
|
+
ingress;
|
|
72
64
|
adminUrl;
|
|
73
65
|
sweepIntervalMs;
|
|
74
66
|
constructor(options) {
|
|
@@ -83,6 +75,18 @@ export class RestateExecution {
|
|
|
83
75
|
sandbox: `${prefix}NylorunSandbox`,
|
|
84
76
|
};
|
|
85
77
|
this.ingressUrl = options.ingressUrl.replace(/\/+$/, "");
|
|
78
|
+
this.ingress = clients.connect({
|
|
79
|
+
url: this.ingressUrl,
|
|
80
|
+
retry: {
|
|
81
|
+
maxAttempts: false,
|
|
82
|
+
maxDuration: SEND_RETRY_MS,
|
|
83
|
+
initialInterval: 100,
|
|
84
|
+
maxInterval: 2000,
|
|
85
|
+
// A just-registered service can take a moment to reach the ingress.
|
|
86
|
+
shouldRetry: (failure) => clients.defaultShouldRetry(failure) ||
|
|
87
|
+
(failure.kind === "response" && failure.status === 404),
|
|
88
|
+
},
|
|
89
|
+
});
|
|
86
90
|
this.adminUrl = options.adminUrl.replace(/\/+$/, "");
|
|
87
91
|
this.sweepIntervalMs = options.sweepIntervalMs ?? 5000;
|
|
88
92
|
}
|
|
@@ -93,28 +97,27 @@ export class RestateExecution {
|
|
|
93
97
|
async wake(tenantId, sessionId, wake) {
|
|
94
98
|
if (!WAKE_REASONS.includes(wake.reason))
|
|
95
99
|
throw new Error(`Unknown wake reason: ${String(wake.reason)}`);
|
|
96
|
-
await this.send(this.names.session, sessionKey(tenantId, sessionId), "advance", { reason: wake.reason }, wake.dedupeKey);
|
|
100
|
+
await this.send(this.names.session, sessionKey(tenantId, sessionId), "advance", { reason: wake.reason }, wake.dedupeKey ?? randomUUID());
|
|
97
101
|
}
|
|
98
102
|
async sandbox(tenantId, sandboxId, signal) {
|
|
99
103
|
const key = sessionKey(tenantId, sandboxId);
|
|
100
104
|
if (signal.kind === "reconcile")
|
|
101
|
-
await this.send(this.names.sandbox, key, "reconcile", {});
|
|
105
|
+
await this.send(this.names.sandbox, key, "reconcile", {}, signal.key ?? randomUUID());
|
|
102
106
|
else
|
|
103
|
-
await this.send(this.names.sandbox, key, "arm", {
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
});
|
|
107
|
+
await this.send(this.names.sandbox, key, "arm", { timer: signal.timer, at: signal.at },
|
|
108
|
+
// Arming a timer for the same time again changes nothing.
|
|
109
|
+
`${signal.timer}@${signal.at}`);
|
|
107
110
|
}
|
|
108
111
|
async timer(tenantId, key, at) {
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
});
|
|
112
|
+
const ms = at.getTime();
|
|
113
|
+
await this.send(this.names.timer, sessionKey(tenantId, key), "set", { at: ms }, `${key}@${ms}`);
|
|
112
114
|
}
|
|
113
115
|
async armSweep(tenantId) {
|
|
114
|
-
|
|
116
|
+
// A key of its own: a later arm (a Host opening the Tenant again) must still run.
|
|
117
|
+
await this.send(this.names.tenant, tenantKey(tenantId), "arm", {}, randomUUID());
|
|
115
118
|
}
|
|
116
119
|
async disarmSweep(tenantId) {
|
|
117
|
-
await this.send(this.names.tenant, tenantKey(tenantId), "disarm", {});
|
|
120
|
+
await this.send(this.names.tenant, tenantKey(tenantId), "disarm", {}, randomUUID());
|
|
118
121
|
}
|
|
119
122
|
/**
|
|
120
123
|
* Readiness: the admin API (`/health`), which `start` registers with, and the
|
|
@@ -366,51 +369,62 @@ export class RestateExecution {
|
|
|
366
369
|
},
|
|
367
370
|
};
|
|
368
371
|
}
|
|
369
|
-
/**
|
|
372
|
+
/**
|
|
373
|
+
* One-way send through the ingress (`@restatedev/restate-sdk-clients`). Every send carries
|
|
374
|
+
* an idempotency key, so the client's retries (`SEND_RETRY_MS`), and a send repeated after
|
|
375
|
+
* its answer was lost, start one invocation: the request's own key where it has one (an
|
|
376
|
+
* outbox row's, a timer's time), or one made for this call.
|
|
377
|
+
*/
|
|
370
378
|
async send(service, key, handler, body, idempotencyKey) {
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
headers,
|
|
379
|
-
body: JSON.stringify(body),
|
|
380
|
-
signal: AbortSignal.timeout(10_000),
|
|
379
|
+
try {
|
|
380
|
+
await this.ingress.send({
|
|
381
|
+
service,
|
|
382
|
+
handler,
|
|
383
|
+
key,
|
|
384
|
+
parameter: body,
|
|
385
|
+
opts: clients.SendOpts.from({ idempotencyKey, timeout: 10_000 }),
|
|
381
386
|
});
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
throw
|
|
390
|
-
|
|
391
|
-
: new PermanentError(error.message);
|
|
392
|
-
});
|
|
387
|
+
}
|
|
388
|
+
catch (error) {
|
|
389
|
+
const message = error instanceof clients.HttpCallError
|
|
390
|
+
? `Restate ingress ${error.status}: ${error.responseText}`
|
|
391
|
+
: error instanceof Error
|
|
392
|
+
? error.message
|
|
393
|
+
: String(error);
|
|
394
|
+
throw new Error(`Restate send to ${service}/${handler} failed: ${message}`, { cause: error });
|
|
395
|
+
}
|
|
393
396
|
}
|
|
394
397
|
/** Registers the Worker endpoint with the admin API, retrying until Restate is up. */
|
|
395
398
|
async register(uri) {
|
|
396
399
|
const force = this.options.forceRegistration ?? true;
|
|
397
|
-
|
|
400
|
+
const registered = await retry(async () => {
|
|
398
401
|
const response = await fetch(`${this.adminUrl}/deployments`, {
|
|
399
402
|
method: "POST",
|
|
400
403
|
headers: { "content-type": "application/json" },
|
|
401
404
|
body: JSON.stringify({ uri, force }),
|
|
402
405
|
signal: AbortSignal.timeout(30_000),
|
|
403
406
|
});
|
|
404
|
-
if (response.ok)
|
|
405
|
-
await response.
|
|
406
|
-
|
|
407
|
-
}
|
|
407
|
+
if (response.ok)
|
|
408
|
+
return (await response.json().catch(() => ({})))
|
|
409
|
+
.services;
|
|
408
410
|
const text = await response.text().catch(() => "");
|
|
409
411
|
const message = `Restate admin ${response.status} registering ${uri}: ${text}`;
|
|
410
412
|
// Discovery failures (Restate cannot reach the endpoint yet) come back
|
|
411
413
|
// as 4xx/5xx; keep trying until the deadline.
|
|
412
414
|
throw new Error(message);
|
|
415
|
+
}, { minMs: 100, maxMs: 2000, deadlineMs: this.options.registrationTimeoutMs ?? 60_000 }).catch((error) => {
|
|
416
|
+
throw new Error(`Restate deployment registration failed: ${error.message}`, {
|
|
417
|
+
cause: error,
|
|
418
|
+
});
|
|
413
419
|
});
|
|
420
|
+
// Without `force`, Restate answers a URL already registered with the deployment it has
|
|
421
|
+
// there, without discovering the endpoint again: one with other services is refused here.
|
|
422
|
+
const names = new Set((registered ?? []).map((service) => service.name));
|
|
423
|
+
const missing = Object.values(this.names).filter((name) => !names.has(name));
|
|
424
|
+
if (registered && missing.length > 0)
|
|
425
|
+
throw new Error(`Restate deployment registration failed: ${uri} is registered with other services ` +
|
|
426
|
+
`(missing ${missing.join(", ")}); register this Worker at a URL of its own ` +
|
|
427
|
+
"(NYLORUN_WORKER_VERSION)");
|
|
414
428
|
}
|
|
415
429
|
}
|
|
416
430
|
/**
|
|
@@ -501,26 +515,6 @@ function tenantKey(tenantId) {
|
|
|
501
515
|
function sqlString(value) {
|
|
502
516
|
return `'${value.replaceAll("'", "''")}'`;
|
|
503
517
|
}
|
|
504
|
-
class PermanentError extends Error {
|
|
505
|
-
}
|
|
506
|
-
async function withRetry(what, timeoutMs, attempt) {
|
|
507
|
-
const deadline = Date.now() + timeoutMs;
|
|
508
|
-
let delay = 100;
|
|
509
|
-
for (;;) {
|
|
510
|
-
try {
|
|
511
|
-
await attempt();
|
|
512
|
-
return;
|
|
513
|
-
}
|
|
514
|
-
catch (error) {
|
|
515
|
-
if (error instanceof PermanentError || Date.now() + delay > deadline)
|
|
516
|
-
throw new Error(`${what} failed: ${error.message}`, {
|
|
517
|
-
cause: error,
|
|
518
|
-
});
|
|
519
|
-
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
520
|
-
delay = Math.min(delay * 2, 2000);
|
|
521
|
-
}
|
|
522
|
-
}
|
|
523
|
-
}
|
|
524
518
|
function sdkLogger(log) {
|
|
525
519
|
return (meta, message, ...rest) => {
|
|
526
520
|
const parts = [message, ...rest].map((part) => part instanceof Error ? part.stack ?? part.message : String(part));
|
|
@@ -57,7 +57,7 @@ export function localBackend(options) {
|
|
|
57
57
|
return reject(signal.reason ?? new Error("Turn cancelled"));
|
|
58
58
|
const child = spawn("/bin/sh", ["-c", request.command], {
|
|
59
59
|
cwd: request.cwd,
|
|
60
|
-
env: { ...env, ...(options.proxyEnv?.() ?? {}) },
|
|
60
|
+
env: { ...env, ...(request.env ?? {}), ...(options.proxyEnv?.() ?? {}) },
|
|
61
61
|
stdio: ["ignore", "pipe", "pipe"],
|
|
62
62
|
// Its own process group, so a timeout or a cancel kills everything it started.
|
|
63
63
|
detached: true,
|
|
@@ -58,6 +58,7 @@
|
|
|
58
58
|
*/
|
|
59
59
|
import { AppendInput, AppendRecord, RangeNotSatisfiableError, S2, S2Error, SeqNumMismatchError, } from "@s2-dev/streamstore";
|
|
60
60
|
import { parseBasin, tenantBasinName, validateBasinPrefix } from "../../streams/basin.js";
|
|
61
|
+
import { sleep } from "../../retry.js";
|
|
61
62
|
import {} from "../../streams/types.js";
|
|
62
63
|
/** Records per unary read while reading history. */
|
|
63
64
|
const HISTORY_PAGE = 1000;
|
|
@@ -357,16 +358,3 @@ function escapeRegExp(text) {
|
|
|
357
358
|
function anySignal(...signals) {
|
|
358
359
|
return AbortSignal.any(signals.filter((s) => s !== undefined));
|
|
359
360
|
}
|
|
360
|
-
function sleep(ms, signal) {
|
|
361
|
-
return new Promise((resolve) => {
|
|
362
|
-
if (signal.aborted)
|
|
363
|
-
return resolve();
|
|
364
|
-
const done = () => {
|
|
365
|
-
clearTimeout(timer);
|
|
366
|
-
signal.removeEventListener("abort", done);
|
|
367
|
-
resolve();
|
|
368
|
-
};
|
|
369
|
-
const timer = setTimeout(done, ms);
|
|
370
|
-
signal.addEventListener("abort", done, { once: true });
|
|
371
|
-
});
|
|
372
|
-
}
|
package/dist/api/components.d.ts
CHANGED
|
@@ -10,6 +10,7 @@ export declare const Rejected: z.ZodObject<{
|
|
|
10
10
|
status: z.ZodLiteral<"rejected">;
|
|
11
11
|
code: z.ZodEnum<{
|
|
12
12
|
connection_missing: "connection_missing";
|
|
13
|
+
credential_conflict: "credential_conflict";
|
|
13
14
|
credential_invalid: "credential_invalid";
|
|
14
15
|
credential_rejected: "credential_rejected";
|
|
15
16
|
credential_required: "credential_required";
|
|
@@ -8376,6 +8377,19 @@ export declare const CreateCredentialRequest: z.ZodObject<{
|
|
|
8376
8377
|
type: z.ZodLiteral<"headers">;
|
|
8377
8378
|
url: z.ZodString;
|
|
8378
8379
|
headers: z.ZodRecord<z.ZodString, z.ZodString>;
|
|
8380
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
8381
|
+
type: z.ZodLiteral<"environment_secret">;
|
|
8382
|
+
secretName: z.ZodString;
|
|
8383
|
+
secretValue: z.ZodString;
|
|
8384
|
+
allowedHosts: z.ZodArray<z.ZodString>;
|
|
8385
|
+
inject: z.ZodOptional<z.ZodObject<{
|
|
8386
|
+
header: z.ZodString;
|
|
8387
|
+
format: z.ZodString;
|
|
8388
|
+
}, z.core.$strict>>;
|
|
8389
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
8390
|
+
type: z.ZodLiteral<"environment_variable">;
|
|
8391
|
+
variableName: z.ZodString;
|
|
8392
|
+
variableValue: z.ZodString;
|
|
8379
8393
|
}, z.core.$strict>], "type">;
|
|
8380
8394
|
}, z.core.$strict>;
|
|
8381
8395
|
export declare const CredentialInfo: z.ZodObject<{
|
|
@@ -8384,10 +8398,20 @@ export declare const CredentialInfo: z.ZodObject<{
|
|
|
8384
8398
|
name: z.ZodString;
|
|
8385
8399
|
type: z.ZodEnum<{
|
|
8386
8400
|
bearer: "bearer";
|
|
8401
|
+
environment_secret: "environment_secret";
|
|
8402
|
+
environment_variable: "environment_variable";
|
|
8387
8403
|
headers: "headers";
|
|
8388
8404
|
}>;
|
|
8389
8405
|
binding: z.ZodObject<{
|
|
8390
|
-
url: z.ZodString
|
|
8406
|
+
url: z.ZodOptional<z.ZodString>;
|
|
8407
|
+
secretName: z.ZodOptional<z.ZodString>;
|
|
8408
|
+
allowedHosts: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
8409
|
+
inject: z.ZodOptional<z.ZodObject<{
|
|
8410
|
+
header: z.ZodString;
|
|
8411
|
+
format: z.ZodString;
|
|
8412
|
+
}, z.core.$strict>>;
|
|
8413
|
+
variableName: z.ZodOptional<z.ZodString>;
|
|
8414
|
+
variableValue: z.ZodOptional<z.ZodString>;
|
|
8391
8415
|
}, z.core.$strict>;
|
|
8392
8416
|
headerNames: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
8393
8417
|
via: z.ZodOptional<z.ZodString>;
|
|
@@ -8404,10 +8428,20 @@ export declare const ListCredentialsResponse: z.ZodObject<{
|
|
|
8404
8428
|
name: z.ZodString;
|
|
8405
8429
|
type: z.ZodEnum<{
|
|
8406
8430
|
bearer: "bearer";
|
|
8431
|
+
environment_secret: "environment_secret";
|
|
8432
|
+
environment_variable: "environment_variable";
|
|
8407
8433
|
headers: "headers";
|
|
8408
8434
|
}>;
|
|
8409
8435
|
binding: z.ZodObject<{
|
|
8410
|
-
url: z.ZodString
|
|
8436
|
+
url: z.ZodOptional<z.ZodString>;
|
|
8437
|
+
secretName: z.ZodOptional<z.ZodString>;
|
|
8438
|
+
allowedHosts: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
8439
|
+
inject: z.ZodOptional<z.ZodObject<{
|
|
8440
|
+
header: z.ZodString;
|
|
8441
|
+
format: z.ZodString;
|
|
8442
|
+
}, z.core.$strict>>;
|
|
8443
|
+
variableName: z.ZodOptional<z.ZodString>;
|
|
8444
|
+
variableValue: z.ZodOptional<z.ZodString>;
|
|
8411
8445
|
}, z.core.$strict>;
|
|
8412
8446
|
headerNames: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
8413
8447
|
via: z.ZodOptional<z.ZodString>;
|
|
@@ -8435,6 +8469,12 @@ export declare const RotateCredentialRequest: z.ZodObject<{
|
|
|
8435
8469
|
}, z.core.$strict>>>;
|
|
8436
8470
|
type: z.ZodLiteral<"headers">;
|
|
8437
8471
|
headers: z.ZodRecord<z.ZodString, z.ZodString>;
|
|
8472
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
8473
|
+
type: z.ZodLiteral<"environment_secret">;
|
|
8474
|
+
secretValue: z.ZodString;
|
|
8475
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
8476
|
+
type: z.ZodLiteral<"environment_variable">;
|
|
8477
|
+
variableValue: z.ZodString;
|
|
8438
8478
|
}, z.core.$strict>], "type">;
|
|
8439
8479
|
}, z.core.$strict>;
|
|
8440
8480
|
export declare const DeletedResponse: z.ZodObject<{
|
package/dist/core/flow-host.d.ts
CHANGED
|
@@ -1,8 +1,7 @@
|
|
|
1
1
|
import type { EffectOutcome } from "@nylorun/core/contracts";
|
|
2
2
|
import { type JsonValue, type WorkflowManifest } from "@nylorun/core/define";
|
|
3
3
|
import type { HostEffect } from "@nylorun/harness/run";
|
|
4
|
-
import type { EffectDoc, Tx
|
|
5
|
-
import { type FlowLimits } from "./limits.js";
|
|
4
|
+
import type { EffectDoc, Tx } from "../store/types.js";
|
|
6
5
|
/** Deterministic agent session id: derive(workflowSessionId, path, …parts). */
|
|
7
6
|
export declare function deriveSessionId(workflowSessionId: string, path: string, ...parts: string[]): string;
|
|
8
7
|
/**
|
|
@@ -97,8 +96,6 @@ export type CascadeCancelPlan = {
|
|
|
97
96
|
/** Linked agent sessions to cancel, deepest path first. */
|
|
98
97
|
readonly agentSessionIds: string[];
|
|
99
98
|
};
|
|
100
|
-
/** Wakes a session after commit (`DurableExecution.wake` through the Tenant context). */
|
|
101
|
-
type Schedule = WakeDelivery;
|
|
102
99
|
/** Effect documents as the flow host writes them. */
|
|
103
100
|
export type FlowEffect = EffectDoc & {
|
|
104
101
|
agentSessionId?: string;
|
|
@@ -135,7 +132,6 @@ export declare function wakeLinkedWorkflow(input: {
|
|
|
135
132
|
readonly failed?: boolean;
|
|
136
133
|
readonly cancelled?: boolean;
|
|
137
134
|
readonly error?: string;
|
|
138
|
-
readonly schedule: Schedule;
|
|
139
135
|
}): Promise<void>;
|
|
140
136
|
/** Pending workflow `agent` effects with a linked session, for the Tenant sweep. */
|
|
141
137
|
export declare function pendingAgentEffects(t: Tx): Promise<FlowEffect[]>;
|
|
@@ -152,7 +148,6 @@ export declare function pendingAgentEffects(t: Tx): Promise<FlowEffect[]>;
|
|
|
152
148
|
export declare function reconcilePendingAgentEffect(input: {
|
|
153
149
|
readonly t: Tx;
|
|
154
150
|
readonly effectId: string;
|
|
155
|
-
readonly schedule: Schedule;
|
|
156
151
|
}): Promise<boolean>;
|
|
157
152
|
/** Path depth for deepest-first cancel ordering. */
|
|
158
153
|
export declare function pathDepth(path: string): number;
|
|
@@ -222,16 +217,3 @@ export declare function foreignInteractionConflict(input: {
|
|
|
222
217
|
message: string;
|
|
223
218
|
ownerSessionId: string;
|
|
224
219
|
} | undefined>;
|
|
225
|
-
/**
|
|
226
|
-
* When concurrency slots free, re-enter the workflow so queued effects can start
|
|
227
|
-
* (status stays `queued` until the effect resolver dispatches them). Returns
|
|
228
|
-
* true when a wake was scheduled.
|
|
229
|
-
*/
|
|
230
|
-
export declare function wakeForQueuedEffects(input: {
|
|
231
|
-
readonly t: Tx;
|
|
232
|
-
readonly workflowSessionId: string;
|
|
233
|
-
readonly turnId: string;
|
|
234
|
-
readonly limits: FlowLimits;
|
|
235
|
-
readonly schedule: Schedule;
|
|
236
|
-
}): Promise<boolean>;
|
|
237
|
-
export {};
|
package/dist/core/flow-host.js
CHANGED
|
@@ -4,8 +4,8 @@
|
|
|
4
4
|
* Every function that touches state takes the caller's transaction `t: Tx` and
|
|
5
5
|
* is async. Nothing here publishes, notifies or schedules directly (seam rule
|
|
6
6
|
* 1): events go through `t.event(...)`, which the store publishes after
|
|
7
|
-
* commit, and wakes go through `t.wake(id, wake
|
|
8
|
-
*
|
|
7
|
+
* commit, and wakes go through `t.wake(id, wake)`, the wake outbox, which the store
|
|
8
|
+
* delivers after commit. Functions
|
|
9
9
|
* that rewrite a session-scoped document lock that session first with
|
|
10
10
|
* `t.lockSession` (a no-op when the caller already holds it).
|
|
11
11
|
*
|
|
@@ -24,9 +24,9 @@
|
|
|
24
24
|
* - `linkedMessageKey(request): string`
|
|
25
25
|
* - `linkedTurnOf(t, agentSessionId, request): Promise<string | undefined>`
|
|
26
26
|
* - `linkedTurnEnd(t, effect, agent): Promise<LinkedTurnEnd | undefined>`
|
|
27
|
-
* - `wakeLinkedWorkflow({ t, agentSessionId, turnId, output?, failed?, cancelled?, error
|
|
27
|
+
* - `wakeLinkedWorkflow({ t, agentSessionId, turnId, output?, failed?, cancelled?, error? }): Promise<void>`
|
|
28
28
|
* - `pendingAgentEffects(t): Promise<FlowEffect[]>`
|
|
29
|
-
* - `reconcilePendingAgentEffect({ t, effectId
|
|
29
|
+
* - `reconcilePendingAgentEffect({ t, effectId }): Promise<boolean>`
|
|
30
30
|
* - `planCancelCascade({ t, workflowSessionId, turnId }): Promise<CascadeCancelPlan>`
|
|
31
31
|
* - `cancelSiblingWork({ t, workflowSessionId, turnId, siblingPaths?, cancelEffectIds? }): Promise<CancelSiblingResult>`
|
|
32
32
|
* - `cancelQueuedEffects({ t, workflowSessionId, turnId }): Promise<string[]>`
|
|
@@ -34,11 +34,10 @@
|
|
|
34
34
|
* - `flowInteractionOf(waits, workflowSessionId, interactionId): { id; kind } | undefined`
|
|
35
35
|
* - `findInteractionOwner({ t, workflowSessionId, interactionId }): Promise<{ sessionId; path } | undefined>`
|
|
36
36
|
* - `foreignInteractionConflict({ t, workflowSessionId, interactionId }): Promise<{ status: 409; message; ownerSessionId } | undefined>`
|
|
37
|
-
* - `wakeForQueuedEffects({ t, workflowSessionId, turnId, limits, schedule }): Promise<boolean>`
|
|
38
37
|
*
|
|
39
|
-
*
|
|
40
|
-
*
|
|
41
|
-
*
|
|
38
|
+
* A wake carries the reason (`linked`, `flow`) and, where the cause has one, a dedupe key.
|
|
39
|
+
* The store delivers it after commit, never inside `t`; one it does not deliver stays in the
|
|
40
|
+
* outbox for the Tenant sweep (`Tx.wake`).
|
|
42
41
|
*
|
|
43
42
|
* An `agent` effect settles only from the linked turn it started. A linked agent session is
|
|
44
43
|
* reused by every iteration of a Loop (its id derives from the workflow and the path), so its
|
|
@@ -51,7 +50,6 @@
|
|
|
51
50
|
*/
|
|
52
51
|
import { createHash } from "node:crypto";
|
|
53
52
|
import { isVerdict } from "@nylorun/core/define";
|
|
54
|
-
import { mayDispatchMore } from "./limits.js";
|
|
55
53
|
/** Deterministic agent session id: derive(workflowSessionId, path, …parts). */
|
|
56
54
|
export function deriveSessionId(workflowSessionId, path, ...parts) {
|
|
57
55
|
const digest = createHash("sha256")
|
|
@@ -140,10 +138,6 @@ export async function recordVerdict(t, request, value) {
|
|
|
140
138
|
...(verdict.data !== undefined ? { data: verdict.data } : {}),
|
|
141
139
|
});
|
|
142
140
|
}
|
|
143
|
-
/** Writes the wake to the outbox; `schedule` delivers it after commit. */
|
|
144
|
-
function requestWake(t, schedule, id, wake) {
|
|
145
|
-
return t.wake(id, wake, schedule);
|
|
146
|
-
}
|
|
147
141
|
/** Active agent turns + tool nodes in flight for one workflow turn. */
|
|
148
142
|
export async function countActiveFlowWork(t, workflowSessionId, turnId) {
|
|
149
143
|
const effects = await t.effectsForTurn(workflowSessionId, turnId);
|
|
@@ -245,7 +239,7 @@ export async function wakeLinkedWorkflow(input) {
|
|
|
245
239
|
await settleAgentEffect({ t, effect, outcome });
|
|
246
240
|
workflow.status = "runnable";
|
|
247
241
|
await t.put("sessions", workflow.id, workflow);
|
|
248
|
-
await
|
|
242
|
+
await t.wake(workflow.id, {
|
|
249
243
|
reason: "linked",
|
|
250
244
|
dedupeKey: `linked:${link.turnId}:${link.effectId}`,
|
|
251
245
|
});
|
|
@@ -288,7 +282,6 @@ export async function reconcilePendingAgentEffect(input) {
|
|
|
288
282
|
: end.status === "failed"
|
|
289
283
|
? { failed: true, error: end.error }
|
|
290
284
|
: { cancelled: true, error: end.error }),
|
|
291
|
-
schedule: input.schedule,
|
|
292
285
|
});
|
|
293
286
|
return true;
|
|
294
287
|
}
|
|
@@ -457,25 +450,3 @@ export async function foreignInteractionConflict(input) {
|
|
|
457
450
|
ownerSessionId: owner.sessionId,
|
|
458
451
|
};
|
|
459
452
|
}
|
|
460
|
-
/**
|
|
461
|
-
* When concurrency slots free, re-enter the workflow so queued effects can start
|
|
462
|
-
* (status stays `queued` until the effect resolver dispatches them). Returns
|
|
463
|
-
* true when a wake was scheduled.
|
|
464
|
-
*/
|
|
465
|
-
export async function wakeForQueuedEffects(input) {
|
|
466
|
-
const { t } = input;
|
|
467
|
-
if (!mayDispatchMore(await countActiveFlowWork(t, input.workflowSessionId, input.turnId), input.limits))
|
|
468
|
-
return false;
|
|
469
|
-
const queued = await t.effectsForTurn(input.workflowSessionId, input.turnId, ["queued"]);
|
|
470
|
-
if (queued.length === 0)
|
|
471
|
-
return false;
|
|
472
|
-
const workflow = await t.lockSession(input.workflowSessionId);
|
|
473
|
-
if (workflow && workflow.status === "waiting") {
|
|
474
|
-
workflow.status = "runnable";
|
|
475
|
-
await t.put("sessions", workflow.id, workflow);
|
|
476
|
-
}
|
|
477
|
-
await requestWake(t, input.schedule, input.workflowSessionId, {
|
|
478
|
-
reason: "flow",
|
|
479
|
-
});
|
|
480
|
-
return true;
|
|
481
|
-
}
|