deepline 0.3.474 → 0.3.476
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/sdk/src/play.ts +2 -2
- package/dist/bundling-sources/sdk/src/release.ts +3 -3
- package/dist/bundling-sources/shared_libs/monitors/provider-monitor-upstream-identity.ts +19 -6
- package/dist/bundling-sources/shared_libs/observability/scheduled-jobs.json +4 -4
- package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts +24 -1
- package/dist/bundling-sources/shared_libs/play-runtime/play-run-recovery-policy.ts +21 -4
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-capacity-policy.ts +12 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-scope.ts +47 -21
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-state.ts +18 -8
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts +163 -40
- package/dist/bundling-sources/shared_libs/plays/authoring-contract.ts +1 -1
- package/dist/cli/auth-2ZY57Q7W.js +41 -0
- package/dist/cli/{auth-QMANU5PV.mjs → auth-A22724UX.mjs} +5 -5
- package/dist/cli/{billing-YJAP4VZN.mjs → billing-IIBEZ6J7.mjs} +6 -6
- package/dist/cli/{billing-WZ5POBZX.js → billing-T5Q6OO6M.js} +79 -79
- package/dist/cli/{chunk-35SZHTRW.mjs → chunk-2WEPQSA2.mjs} +5 -5
- package/dist/cli/{chunk-MXUJVCKV.mjs → chunk-2YHDNHRR.mjs} +4 -4
- package/dist/cli/{chunk-V4ILQ2DS.mjs → chunk-3A7TDUZV.mjs} +1 -1
- package/dist/cli/{chunk-BB4RZMWA.mjs → chunk-4WMIMSSI.mjs} +5 -5
- package/dist/cli/{chunk-ZRWQKTSX.mjs → chunk-4YWYLHED.mjs} +3 -3
- package/dist/cli/{chunk-ORQYKA4I.js → chunk-5AEEINTU.js} +4 -4
- package/dist/cli/{chunk-3NRZFMX4.js → chunk-5FBCMLJW.js} +6 -6
- package/dist/cli/{chunk-XNP7XTBN.js → chunk-5PYSYW63.js} +26 -26
- package/dist/cli/{chunk-PGKELBIY.js → chunk-6YZYCKWD.js} +51 -51
- package/dist/cli/{chunk-RXITRC37.js → chunk-7HAUVMRF.js} +3 -3
- package/dist/cli/{chunk-JCNJQH6X.js → chunk-A6BETJ2A.js} +3 -3
- package/dist/cli/{chunk-STLJYWUC.js → chunk-AS3VECCY.js} +7 -7
- package/dist/cli/{chunk-LHOEWAVD.js → chunk-B7PEXXRW.js} +53 -53
- package/dist/cli/{chunk-IK3VOGDR.js → chunk-CBVCCXIY.js} +21 -21
- package/dist/cli/{chunk-JEYX3345.mjs → chunk-CFSCHLNQ.mjs} +3 -3
- package/dist/cli/{chunk-3MZPWIOE.mjs → chunk-D7EQ4PI5.mjs} +1 -1
- package/dist/cli/{chunk-XP7FAYLN.js → chunk-EBWYNNTJ.js} +5 -5
- package/dist/cli/{chunk-DQZBERDL.mjs → chunk-F5PYBZ2H.mjs} +1 -1
- package/dist/cli/{chunk-AIPTW3EK.js → chunk-FK7SBMSZ.js} +3 -3
- package/dist/cli/{chunk-Z27NSLPC.mjs → chunk-FVPT44IJ.mjs} +1 -1
- package/dist/cli/{chunk-RGIQ5ZJX.mjs → chunk-H3GOAZHV.mjs} +2 -2
- package/dist/cli/{chunk-AWJKYS25.js → chunk-HSANVBVL.js} +2 -2
- package/dist/cli/{chunk-JSXMJSBE.js → chunk-IC3BZJR7.js} +8 -8
- package/dist/cli/{chunk-MNK4ZK2C.mjs → chunk-J4FCTFQS.mjs} +10 -10
- package/dist/cli/{chunk-MKSB2FTU.mjs → chunk-JU2KBLOR.mjs} +97 -12
- package/dist/cli/{chunk-EIKONWW7.mjs → chunk-KCP6XNRR.mjs} +1 -1
- package/dist/cli/{chunk-YKNULB7V.js → chunk-KGLIDLJ2.js} +5 -5
- package/dist/cli/{chunk-N62MS3XJ.js → chunk-L45IECBZ.js} +6 -6
- package/dist/cli/{chunk-7IYOWIEJ.mjs → chunk-LJFIUT5R.mjs} +2 -2
- package/dist/cli/{chunk-ZGC2SUAA.mjs → chunk-LZCYDU6Q.mjs} +2 -2
- package/dist/cli/{chunk-3ODBCZLL.js → chunk-MYYOCJZ2.js} +93 -93
- package/dist/cli/{chunk-DKR7QPEO.js → chunk-MZSLFWOT.js} +1 -1
- package/dist/cli/{chunk-26CZVDQN.mjs → chunk-NPNQCPFP.mjs} +2 -2
- package/dist/cli/{chunk-UKX4FESF.mjs → chunk-NREO7TT4.mjs} +2 -2
- package/dist/cli/{chunk-KKWQ24VM.mjs → chunk-NTJDZURV.mjs} +2 -2
- package/dist/cli/{chunk-4C34LGI4.js → chunk-PGNLD6L6.js} +10 -10
- package/dist/cli/{chunk-G3UHAYRX.js → chunk-TEJSRJNH.js} +8 -8
- package/dist/cli/{chunk-UGYLGT7B.mjs → chunk-UHTJWDYB.mjs} +3 -3
- package/dist/cli/{chunk-EK2TAZJZ.js → chunk-ULKGYVYP.js} +2 -2
- package/dist/cli/{chunk-DUGZKPRK.mjs → chunk-UYA7LJGC.mjs} +2 -2
- package/dist/cli/{chunk-YVJ6IWVD.mjs → chunk-VKBZLV2K.mjs} +1 -1
- package/dist/cli/{chunk-YFDTF2XC.js → chunk-VLYUFZUT.js} +9 -9
- package/dist/cli/{chunk-THDXVSNL.mjs → chunk-VPAVNEDQ.mjs} +4 -4
- package/dist/cli/{chunk-RLQIGBFK.js → chunk-W3BHZUPL.js} +3 -3
- package/dist/cli/{chunk-5QS4T4KR.js → chunk-WFVEMDQL.js} +2 -2
- package/dist/cli/{chunk-475V572K.js → chunk-WTPRRKZA.js} +131 -46
- package/dist/cli/{chunk-CXBEZAPU.js → chunk-YEJDFZJP.js} +16 -16
- package/dist/cli/{chunk-JJVYUKLY.mjs → chunk-YQOWMQEW.mjs} +1 -1
- package/dist/cli/{chunk-YK64UZXN.mjs → chunk-ZOXV6WFI.mjs} +1 -1
- package/dist/cli/{chunk-YX762HFK.mjs → chunk-ZV4OWYLZ.mjs} +2 -2
- package/dist/cli/command-docs.js +2 -2
- package/dist/cli/command-docs.mjs +1 -1
- package/dist/cli/{csv-ENKCFCSM.js → csv-FXOQ2PBE.js} +13 -13
- package/dist/cli/{csv-QCLQOIEA.mjs → csv-GPVCETS2.mjs} +4 -4
- package/dist/cli/{db-P25LGP33.mjs → db-F5UJTBGP.mjs} +6 -6
- package/dist/cli/{db-5TLBZDT3.js → db-LXYV4GPZ.js} +27 -27
- package/dist/cli/{describe-W2H5DHR6.mjs → describe-GYWCI7I6.mjs} +17 -17
- package/dist/cli/describe-IB7SUIUT.js +48 -0
- package/dist/cli/describe-core.d.mts +3 -3
- package/dist/cli/describe-core.d.ts +3 -3
- package/dist/cli/{executions-VYZCYUBR.mjs → executions-6NL2YW76.mjs} +6 -6
- package/dist/cli/{executions-YHFAWJOJ.js → executions-6SJKTY2L.js} +8 -8
- package/dist/cli/{feedback-K2FWYYLZ.mjs → feedback-7AMQCNTB.mjs} +4 -4
- package/dist/cli/{feedback-RJF3QSBB.js → feedback-E3HULSQD.js} +8 -8
- package/dist/cli/{git-S3APUPLL.mjs → git-74IS2EFH.mjs} +5 -5
- package/dist/cli/git-A2QIIMW5.js +25 -0
- package/dist/cli/index.js +139 -139
- package/dist/cli/index.mjs +41 -41
- package/dist/cli/{legacy-noops-ICYZACTW.mjs → legacy-noops-52FFRTRZ.mjs} +5 -5
- package/dist/cli/{legacy-noops-3SWNRE7Z.js → legacy-noops-ALWUQPKS.js} +8 -8
- package/dist/cli/{monitors-DEZCEQLB.mjs → monitors-VTJIWWW2.mjs} +7 -7
- package/dist/cli/{monitors-VRIWTRPG.js → monitors-Z4ADOGTY.js} +91 -91
- package/dist/cli/{org-NX752DTE.js → org-2BTEKDF5.js} +72 -72
- package/dist/cli/{org-IQW5ZIAO.mjs → org-7GU6H7JL.mjs} +6 -6
- package/dist/cli/{pages-5WQRVVKX.mjs → pages-I6PFQGEU.mjs} +6 -6
- package/dist/cli/{pages-YIBWYGVS.js → pages-MG36E7RY.js} +17 -17
- package/dist/cli/{play-WOMJXS4O.mjs → play-7KFSLJX2.mjs} +18 -18
- package/dist/cli/{play-3EXNH3HD.js → play-VN52S3RV.js} +321 -321
- package/dist/cli/{providers-XDVPYRTS.mjs → providers-D46JDEL2.mjs} +6 -6
- package/dist/cli/{providers-C2WYSR2Y.js → providers-W5XLII66.js} +9 -9
- package/dist/cli/{quickstart-YMSIL2TK.mjs → quickstart-4GTXBZHY.mjs} +4 -4
- package/dist/cli/{quickstart-7RABZF66.js → quickstart-TIZAC63S.js} +6 -6
- package/dist/cli/{run-5QVVWFGL.js → run-FOVEMYYX.js} +36 -36
- package/dist/cli/{run-A4JD6TAR.mjs → run-LE3D4LVU.mjs} +22 -22
- package/dist/cli/{search-2TOSF2G3.mjs → search-6YL6WJMQ.mjs} +7 -7
- package/dist/cli/search-HN6SVDQU.js +32 -0
- package/dist/cli/{secrets-Q7UVOSNM.js → secrets-CSTTTMZH.js} +15 -15
- package/dist/cli/{secrets-BUN4ORE7.mjs → secrets-D66U6RTT.mjs} +6 -6
- package/dist/cli/{sessions-NXQOQ4HX.js → sessions-45M2NRAA.js} +6 -6
- package/dist/cli/{sessions-IAOORI6X.mjs → sessions-XIO3OUER.mjs} +5 -5
- package/dist/cli/{settings-WD4KSY5S.js → settings-N42CVKSK.js} +78 -78
- package/dist/cli/{settings-BMI373AI.mjs → settings-R33X2D5T.mjs} +11 -11
- package/dist/cli/{setup-55VIWAWR.mjs → setup-CBBC4FQP.mjs} +9 -9
- package/dist/cli/{setup-3IA7BVPE.js → setup-HREDVNF2.js} +44 -44
- package/dist/cli/{skills-GKITMBTM.mjs → skills-4X6WS2PT.mjs} +9 -7
- package/dist/cli/skills-QZ6HZX2T.js +33 -0
- package/dist/cli/tools-KFIH7UWB.js +55 -0
- package/dist/cli/{tools-XHCFC7GP.mjs → tools-YR46YFR2.mjs} +16 -16
- package/dist/cli/{update-SWB24YKE.mjs → update-5CYAZX5X.mjs} +9 -9
- package/dist/cli/update-VZLEJLWJ.js +33 -0
- package/dist/cli/{workflow-5QPMFI3Z.js → workflow-M7AA722W.js} +17 -17
- package/dist/cli/{workflow-XO4CKUA7.mjs → workflow-NKQ3CWPK.mjs} +7 -7
- package/dist/{client-Bgl0_zXi.d.mts → client-AjmfA5nR.d.mts} +1 -1
- package/dist/{client-_ah6IuhK.d.ts → client-DMyY-QL8.d.ts} +1 -1
- package/dist/{client-core-C-ZF5vh9.d.ts → client-core-C705vFyV.d.ts} +1 -1
- package/dist/{client-core-Bfsedm0_.d.mts → client-core-CpBbvLEX.d.mts} +1 -1
- package/dist/{compiler-manifest-CCOqDGjU.d.mts → compiler-manifest-hqIAukR0.d.mts} +1 -1
- package/dist/{compiler-manifest-CCOqDGjU.d.ts → compiler-manifest-hqIAukR0.d.ts} +1 -1
- package/dist/{errors-Cy3kPnsi.d.ts → errors-CRO2qQ6k.d.ts} +1 -1
- package/dist/{errors-BLccD7tG.d.mts → errors-DIHVetNY.d.mts} +1 -1
- package/dist/{index-D5e4h1-H.d.ts → index-B8DVoeYt.d.ts} +1 -1
- package/dist/{index-4-j44HuT.d.mts → index-DVZ_X4Gb.d.mts} +1 -1
- package/dist/index.d.mts +9 -9
- package/dist/index.d.ts +9 -9
- package/dist/index.js +9 -9
- package/dist/index.mjs +6 -6
- package/dist/install-integrity.json +110 -110
- package/dist/plays/bundle-play-file.d.mts +3 -3
- package/dist/plays/bundle-play-file.d.ts +3 -3
- package/dist/plays/bundle-play-file.mjs +1 -1
- package/dist/plays/play-description.d.mts +2 -2
- package/dist/plays/play-description.d.ts +2 -2
- package/dist/plays/run-request.d.mts +2 -2
- package/dist/plays/run-request.d.ts +2 -2
- package/dist/release.d.mts +3 -3
- package/dist/release.d.ts +3 -3
- package/dist/release.js +2 -2
- package/dist/release.mjs +1 -1
- package/dist/server.d.mts +4 -4
- package/dist/server.d.ts +4 -4
- package/dist/server.js +5 -5
- package/dist/server.mjs +3 -3
- package/package.json +1 -1
- package/dist/cli/auth-O6KAH543.js +0 -41
- package/dist/cli/describe-44L7OBGM.js +0 -48
- package/dist/cli/git-F3EI5DY5.js +0 -25
- package/dist/cli/search-772R7YHI.js +0 -32
- package/dist/cli/skills-PUUTFMP3.js +0 -31
- package/dist/cli/tools-3WX7TUUG.js +0 -55
- package/dist/cli/update-PC3CLODV.js +0 -33
|
@@ -185,8 +185,8 @@ export type PlayFetchResponse = PlayAuthoringFetchResponse;
|
|
|
185
185
|
* The changed row is delivered as the listener event's `after` in the play function's input.
|
|
186
186
|
*
|
|
187
187
|
* The default Play runtime is 30 minutes with the `standard` 1 GiB / 1 CPU
|
|
188
|
-
* profile.
|
|
189
|
-
* `
|
|
188
|
+
* profile. Select `large` (4 GiB / 2 CPU) or `xlarge` (8 GiB / 4 CPU)
|
|
189
|
+
* through `runtime.size` when a step needs more memory.
|
|
190
190
|
* Compute pricing follows the workspace plan; use `deepline billing quote`
|
|
191
191
|
* for the session ceiling.
|
|
192
192
|
* Set `runtime.timeout` for longer bounded runs; duration values are whole
|
|
@@ -205,7 +205,7 @@ export const SDK_RELEASE = {
|
|
|
205
205
|
// getters keep their established compatibility behavior.
|
|
206
206
|
// 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
|
|
207
207
|
// 0.3.90 is the first deliberately versioned SDK release for API v3.
|
|
208
|
-
version: '0.3.
|
|
208
|
+
version: '0.3.476',
|
|
209
209
|
updateSummary: {
|
|
210
210
|
introducedIn: '0.3.90',
|
|
211
211
|
summary:
|
|
@@ -252,9 +252,9 @@ export const SDK_RELEASE = {
|
|
|
252
252
|
},
|
|
253
253
|
{
|
|
254
254
|
command: 'search',
|
|
255
|
-
introducedIn: '0.3.
|
|
255
|
+
introducedIn: '0.3.287',
|
|
256
256
|
reason:
|
|
257
|
-
'deepline search was added in SDK CLI 0.3.
|
|
257
|
+
'deepline search was added in SDK CLI 0.3.287 as the one search across tools, monitors, and plays. On an older version run deepline tools search or deepline plays search, or update.',
|
|
258
258
|
},
|
|
259
259
|
{
|
|
260
260
|
command: 'notifications',
|
|
@@ -46,8 +46,9 @@ type ProviderMonitorUpstreamIdentityDefinition = {
|
|
|
46
46
|
* Payload fields that never change which findings the provider resource
|
|
47
47
|
* delivers. `updates_since` only sets how far back the first delivery
|
|
48
48
|
* replays, so two monitors that differ only there watch the same feed.
|
|
49
|
-
*
|
|
50
|
-
*
|
|
49
|
+
* New claim keys omit them. Keys stored before that still carry them until
|
|
50
|
+
* the `normalizeProviderMonitorIdentityClaimKeys` migration rewrites them;
|
|
51
|
+
* readers resolve both through `canonicalProviderMonitorIdentityClaimKey`.
|
|
51
52
|
*/
|
|
52
53
|
nonConfigurationFields?: readonly string[];
|
|
53
54
|
};
|
|
@@ -171,9 +172,11 @@ export function providerMonitorUpstreamIdentitySignature(
|
|
|
171
172
|
/**
|
|
172
173
|
* Exact claim identity used by the Convex conflict index. The provider's
|
|
173
174
|
* declared scope remains visible in `scopeKey`; providers that allow
|
|
174
|
-
* configuration variants add every non-scope payload field to the claim key
|
|
175
|
-
* Keeping the old scope JSON as the prefix
|
|
176
|
-
* created before this distinction was
|
|
175
|
+
* configuration variants add every non-scope payload field to the claim key,
|
|
176
|
+
* except `nonConfigurationFields`. Keeping the old scope JSON as the prefix
|
|
177
|
+
* lets readers recognize claims created before this distinction was
|
|
178
|
+
* introduced. The result is always canonical
|
|
179
|
+
* (`canonicalProviderMonitorIdentityClaimKey` returns it unchanged).
|
|
177
180
|
*/
|
|
178
181
|
export function providerMonitorUpstreamIdentityClaimKey(
|
|
179
182
|
tool: string,
|
|
@@ -206,9 +209,10 @@ export function providerMonitorUpstreamIdentityClaimKey(
|
|
|
206
209
|
: []),
|
|
207
210
|
]),
|
|
208
211
|
);
|
|
212
|
+
const ignored = new Set(definition.nonConfigurationFields ?? []);
|
|
209
213
|
const configuration = Object.fromEntries(
|
|
210
214
|
Object.entries(payload ?? {}).filter(
|
|
211
|
-
([field]) => !scopeAndAliases.has(field),
|
|
215
|
+
([field]) => !scopeAndAliases.has(field) && !ignored.has(field),
|
|
212
216
|
),
|
|
213
217
|
);
|
|
214
218
|
if (Object.keys(configuration).length === 0) return scopeKey;
|
|
@@ -275,6 +279,15 @@ export function canonicalProviderMonitorIdentityClaimKey(
|
|
|
275
279
|
return `${parts.scopeKey}${CONFIGURATION_SEPARATOR}${stableJson(configuration)}`;
|
|
276
280
|
}
|
|
277
281
|
|
|
282
|
+
/**
|
|
283
|
+
* The scope part of a stored claim key: the whole key when it is scope-only.
|
|
284
|
+
* Every key for one provider scope starts with this exact JSON object, so it
|
|
285
|
+
* bounds an index range over that scope's claims.
|
|
286
|
+
*/
|
|
287
|
+
export function providerMonitorIdentityClaimScopeKey(identityKey: string) {
|
|
288
|
+
return splitClaimKey(identityKey)?.scopeKey ?? identityKey;
|
|
289
|
+
}
|
|
290
|
+
|
|
278
291
|
/** Whether a claim key carries configuration beyond its scope. */
|
|
279
292
|
export function providerMonitorIdentityClaimKeyHasConfiguration(
|
|
280
293
|
identityKey: string,
|
|
@@ -241,7 +241,7 @@
|
|
|
241
241
|
"id": "github.eval-agent-cli-matrix",
|
|
242
242
|
"name": "Eval Agent CLI Matrix",
|
|
243
243
|
"scheduler": "github_actions",
|
|
244
|
-
"cadence": "0 7 * * *",
|
|
244
|
+
"cadence": "0 7 * * *; 0 22 * * 4; 0 3 * * 1,3,5",
|
|
245
245
|
"timezone": "America/New_York",
|
|
246
246
|
"source": ".github/workflows/eval-agent-cli-matrix.yml",
|
|
247
247
|
"locator": "schedule:",
|
|
@@ -265,9 +265,9 @@
|
|
|
265
265
|
"locator": "schedule:",
|
|
266
266
|
"owner": "platform",
|
|
267
267
|
"criticality": "maintenance",
|
|
268
|
-
"purpose": "Reconcile completed PR Preview Machines and release capacity only after the per-app lease fence allows it. Its named-estate lane plans every named Preview estate (eval reuse keys, runtime up preview branch lanes, one-off namespaces) with `bun scripts/fly-preview-estate-lifecycle.ts --report --named-reclaim-plan` and, only on a workflow_dispatch with reclaim_named_estates=true, scales at most 8 planned estates per pass (interrupted reclaims first, then reclaimable estates idle 24 h by lease, release or reclaim evidence) to zero through the claimed runtime-preview-namespace.yml scale-to-zero, recording the start and lane app before the seam, retiring the release lane and recording reclaimed history; scheduled passes only report it.",
|
|
269
|
-
"successDefinition": "Fly inventory is reconciled and eligible terminal/closed PR capacity is released; every named estate is classified without a read failure, and a dispatched named reclaim leaves each target at zero Machines with its release lane retired and reclaimed history written. No eligible app or estate is a valid no-op.",
|
|
270
|
-
"failureDefinition": "Inventory, lease resolution, scale-down, or empty-app cleanup fails or is ambiguous; a named-estate Fly or authority read fails; a named reclaim is refused inside its claim (active owner, used since discovery, ownership or lane-app refusal) or any role keeps a Machine; or any named reclaim is incomplete (started but its release lane was never retired), which fails every pass until a dispatch finishes it.",
|
|
268
|
+
"purpose": "Reconcile completed PR Preview Machines and release capacity only after the per-app lease fence allows it. Its named-estate lane plans every named Preview estate (eval reuse keys, runtime up preview branch lanes, one-off namespaces) with `bun scripts/fly-preview-estate-lifecycle.ts --report --named-reclaim-plan` and, only on a workflow_dispatch with reclaim_named_estates=true, scales at most 8 planned estates per pass (interrupted reclaims first, then reclaimable estates idle 24 h by lease, release or reclaim evidence) to zero through the claimed runtime-preview-namespace.yml scale-to-zero, recording the start and lane app before the seam, retiring the release lane and recording reclaimed history; scheduled passes only report it. Each pass also returns the deeplinev1/load PlanetScale branch to its parked size when no Customer Workload Suite run is active on load-tests/framework, in case a load test's own teardown never ran.",
|
|
269
|
+
"successDefinition": "Fly inventory is reconciled and eligible terminal/closed PR capacity is released; every named estate is classified without a read failure, and a dispatched named reclaim leaves each target at zero Machines with its release lane retired and reclaimed history written. No eligible app or estate is a valid no-op. The load database is parked, a change toward that size is in flight, or an active Customer Workload Suite run on load-tests/framework may be holding it.",
|
|
270
|
+
"failureDefinition": "Inventory, lease resolution, scale-down, or empty-app cleanup fails or is ambiguous; a named-estate Fly or authority read fails; a named reclaim is refused inside its claim (active owner, used since discovery, ownership or lane-app refusal) or any role keeps a Machine; or any named reclaim is incomplete (started but its release lane was never retired), which fails every pass until a dispatch finishes it; or reading or parking the load database fails.",
|
|
271
271
|
"startGraceMs": 600000,
|
|
272
272
|
"runTimeoutMs": 3600000,
|
|
273
273
|
"failureAlert": "github_workflow_run",
|
package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts
CHANGED
|
@@ -790,6 +790,13 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
|
|
|
790
790
|
): void {
|
|
791
791
|
if (retryAtMs <= (this.localProviderRetryAtMs.get(bucketId) ?? 0)) return;
|
|
792
792
|
this.localProviderRetryAtMs.set(bucketId, retryAtMs);
|
|
793
|
+
const cadence = this.validatedDispatchCadences.get(bucketId);
|
|
794
|
+
const serverOffsetMs = this.serverClockOffsetsMs.get(bucketId) ?? 0;
|
|
795
|
+
// A new hold fences old tickets and their dispatch debt. Keep debt from
|
|
796
|
+
// sends after an expired hold; their physical spacing is still current.
|
|
797
|
+
if (cadence && cadence.lastAtMs + serverOffsetMs <= retryAtMs) {
|
|
798
|
+
this.validatedDispatchCadences.delete(bucketId);
|
|
799
|
+
}
|
|
793
800
|
for (const waiting of this.scheduledPermitWaits.get(bucketId) ?? []) {
|
|
794
801
|
if (waiting.retryAtMsAtGrant < retryAtMs) waiting.invalidate();
|
|
795
802
|
}
|
|
@@ -1626,6 +1633,14 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
|
|
|
1626
1633
|
scopeRetryAtMsAtGrant,
|
|
1627
1634
|
dispatchValidationSupported,
|
|
1628
1635
|
] = scheduled;
|
|
1636
|
+
// Legacy grants have no shared hold snapshot. Capture their local fence
|
|
1637
|
+
// when assigned, while keeping the signed gateway proof unchanged.
|
|
1638
|
+
const localRetryAtMsAtGrant = this.validationScopes.has(bucketId)
|
|
1639
|
+
? scopeRetryAtMsAtGrant
|
|
1640
|
+
: Math.max(
|
|
1641
|
+
scopeRetryAtMsAtGrant,
|
|
1642
|
+
this.localProviderRetryAtMs.get(bucketId) ?? 0,
|
|
1643
|
+
);
|
|
1629
1644
|
let settled = false;
|
|
1630
1645
|
const refund = async (): Promise<void> => {
|
|
1631
1646
|
if (!reservationId) {
|
|
@@ -1732,7 +1747,15 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
|
|
|
1732
1747
|
});
|
|
1733
1748
|
}
|
|
1734
1749
|
if (waitMs > 0) {
|
|
1735
|
-
|
|
1750
|
+
if (
|
|
1751
|
+
!(await this.waitForScheduledPermit(
|
|
1752
|
+
bucketId,
|
|
1753
|
+
localRetryAtMsAtGrant,
|
|
1754
|
+
waitMs,
|
|
1755
|
+
signal,
|
|
1756
|
+
))
|
|
1757
|
+
)
|
|
1758
|
+
return false;
|
|
1736
1759
|
continue;
|
|
1737
1760
|
}
|
|
1738
1761
|
this.validatedDispatchCadences.set(bucketId, {
|
|
@@ -72,11 +72,14 @@ export type PlayRunRecoveryDecision = Readonly<{
|
|
|
72
72
|
export class PlayRunRecoveryError extends Error {
|
|
73
73
|
readonly decision: PlayRunRecoveryDecision;
|
|
74
74
|
readonly cause?: unknown;
|
|
75
|
+
/** Log-only provider facts; see `playRunRecoveryLogDetail`. */
|
|
76
|
+
readonly recoveryLogDetail?: unknown;
|
|
75
77
|
|
|
76
78
|
constructor(input: {
|
|
77
79
|
decision: PlayRunRecoveryDecision;
|
|
78
80
|
message: string;
|
|
79
81
|
cause?: unknown;
|
|
82
|
+
recoveryLogDetail?: unknown;
|
|
80
83
|
}) {
|
|
81
84
|
super(input.message);
|
|
82
85
|
this.name = PLAY_RUN_RECOVERY_ERROR_NAME;
|
|
@@ -84,6 +87,9 @@ export class PlayRunRecoveryError extends Error {
|
|
|
84
87
|
if (input.cause !== undefined) {
|
|
85
88
|
this.cause = input.cause;
|
|
86
89
|
}
|
|
90
|
+
if (input.recoveryLogDetail !== undefined) {
|
|
91
|
+
this.recoveryLogDetail = input.recoveryLogDetail;
|
|
92
|
+
}
|
|
87
93
|
}
|
|
88
94
|
}
|
|
89
95
|
|
|
@@ -106,7 +112,7 @@ export type PlayRunRecoveryLogDetail = Readonly<
|
|
|
106
112
|
>;
|
|
107
113
|
|
|
108
114
|
/**
|
|
109
|
-
* Provider facts an adapter attached to
|
|
115
|
+
* Provider facts an adapter attached to a recovery error or its cause, for the
|
|
110
116
|
* scheduler's defer log only. The decision's `reason` stays the single durable
|
|
111
117
|
* recovery fact: this detail is never persisted, never returned to a caller,
|
|
112
118
|
* and never read to choose a recovery action. It is bounded here, at the edge
|
|
@@ -119,9 +125,11 @@ export function playRunRecoveryLogDetail(
|
|
|
119
125
|
try {
|
|
120
126
|
if (!isPlayRunRecoveryError(error)) return null;
|
|
121
127
|
const cause: unknown = error.cause;
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
128
|
+
const supplied =
|
|
129
|
+
error.recoveryLogDetail ??
|
|
130
|
+
(cause && typeof cause === 'object'
|
|
131
|
+
? (cause as { recoveryLogDetail?: unknown }).recoveryLogDetail
|
|
132
|
+
: undefined);
|
|
125
133
|
if (!supplied || typeof supplied !== 'object' || Array.isArray(supplied)) {
|
|
126
134
|
return null;
|
|
127
135
|
}
|
|
@@ -203,6 +211,14 @@ export function recoveryForModalSandboxCreateFailure(
|
|
|
203
211
|
const code = errorCode(error);
|
|
204
212
|
const status = errorStatus(error);
|
|
205
213
|
const message = messageOf(error);
|
|
214
|
+
// Only this rejection carries Modal's own words. Once it opens the circuit,
|
|
215
|
+
// later runs defer without calling Modal, and every reason here groups
|
|
216
|
+
// several provider conditions.
|
|
217
|
+
const recoveryLogDetail = {
|
|
218
|
+
providerCode: code,
|
|
219
|
+
providerStatus: status,
|
|
220
|
+
providerMessage: message,
|
|
221
|
+
};
|
|
206
222
|
const resourceExhausted =
|
|
207
223
|
code === 8 ||
|
|
208
224
|
(typeof code === 'string' && /RESOURCE_EXHAUSTED/i.test(code)) ||
|
|
@@ -218,6 +234,7 @@ export function recoveryForModalSandboxCreateFailure(
|
|
|
218
234
|
message:
|
|
219
235
|
'Modal sandbox capacity is unavailable before customer execution began.',
|
|
220
236
|
cause: error,
|
|
237
|
+
recoveryLogDetail,
|
|
221
238
|
});
|
|
222
239
|
}
|
|
223
240
|
|
|
@@ -56,8 +56,20 @@ export const RUNTIME_CAPACITY_POLICY = {
|
|
|
56
56
|
maxScaleDownMachinesPerReconcile: 2,
|
|
57
57
|
/** Empty relay capacity must be quiet before it is reclaimed. */
|
|
58
58
|
scaleDownQuietMs: 120_000,
|
|
59
|
+
/**
|
|
60
|
+
* Pooled Postgres clients per relay Machine when
|
|
61
|
+
* DEEPLINE_RUNTIME_GATEWAY_POOL_MAX_CLIENTS sets none, as in production.
|
|
62
|
+
* packages/runtime-gateway/server.ts explains the budget.
|
|
63
|
+
*/
|
|
64
|
+
poolMaxClients: 6,
|
|
59
65
|
},
|
|
60
66
|
receiptGateway: {
|
|
67
|
+
/**
|
|
68
|
+
* Pooled Postgres clients per receipt-gateway Machine when
|
|
69
|
+
* DEEPLINE_RUNTIME_GATEWAY_POOL_MAX_CLIENTS sets none, as in production.
|
|
70
|
+
* packages/runtime-gateway/server.ts explains the budget.
|
|
71
|
+
*/
|
|
72
|
+
poolMaxClients: 8,
|
|
61
73
|
admissionTimeoutMs: 10_000,
|
|
62
74
|
/** End identical receipt-row work before the caller can overlap a retry. */
|
|
63
75
|
claimOperationTimeoutMs: 20_000,
|
package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-scope.ts
CHANGED
|
@@ -72,35 +72,57 @@ export function acquireRateScopeSql(
|
|
|
72
72
|
THEN CASE WHEN ${productiveRecovery} THEN ${AIMD_RECOVERY_STEP_MS}
|
|
73
73
|
ELSE ${sparseRecoveryInterval} END
|
|
74
74
|
ELSE LEAST(${floorRecoveryInterval}, CEIL(1000.0 / ${refillRate})) END`
|
|
75
|
-
:
|
|
75
|
+
: `${AIMD_RECOVERY_STEP_MS}`;
|
|
76
76
|
// A local no-progress pause remains active after the provider's shorter
|
|
77
77
|
// Retry-After. Only time beyond every outstanding dispatch/hold is idle.
|
|
78
78
|
const observationWasIdle = `${successesParameter}::int = 0
|
|
79
79
|
AND nv.now_ms - GREATEST(h.refilled_at_ms, h.next_dispatch_at_ms, h.retry_at_ms, h.cool_until_ms)
|
|
80
80
|
> GREATEST(1000, 2000.0 / ${refillRate})`;
|
|
81
|
-
// A denied refill may fold a completion before the next
|
|
82
|
-
//
|
|
83
|
-
//
|
|
84
|
-
//
|
|
85
|
-
const recoverySuccess =
|
|
86
|
-
|
|
87
|
-
? `(${successesParameter}::int > 0 OR (
|
|
81
|
+
// A denied refill may fold a completion before the next recovery interval.
|
|
82
|
+
// Keep it available for one later ramp rather than requiring another call at
|
|
83
|
+
// the old pace. The baseline consumes it once. Unknown-provider upward probes
|
|
84
|
+
// above useful seed capacity retain their separate observation policy.
|
|
85
|
+
const recoverySuccess = probeBaselineSupported
|
|
86
|
+
? `(${successesParameter}::int > 0 OR (
|
|
88
87
|
h.seen_backpressure
|
|
89
|
-
AND ${refillRate} < ${DEFAULT_UNKNOWN_PROVIDER_FAST_RECOVERY_RPS}
|
|
88
|
+
AND (${!unknownProvider} OR ${refillRate} < ${DEFAULT_UNKNOWN_PROVIDER_FAST_RECOVERY_RPS})
|
|
90
89
|
AND h.rate_probe_started_at_ms >= h.last_ramp_at_ms
|
|
91
90
|
AND h.rate_probe_started_at_ms >= h.rate_observation_started_at_ms
|
|
92
91
|
AND h.rate_successes > h.rate_probe_successes_at_ramp
|
|
93
92
|
AND NOT (${observationWasIdle})
|
|
94
93
|
))`
|
|
95
|
-
|
|
94
|
+
: `${successesParameter}::int > 0`;
|
|
95
|
+
// Known-provider feedback can arrive before its recovery interval. Wake at
|
|
96
|
+
// that opportunity instead of sleeping an entire old, slow token interval;
|
|
97
|
+
// the baseline below consumes each confirmed success for at most one ramp.
|
|
98
|
+
const pendingRecoveryWaitMs =
|
|
99
|
+
!unknownProvider && probeBaselineSupported
|
|
100
|
+
? `CASE WHEN ${recoverySuccess}
|
|
101
|
+
AND h.seen_backpressure
|
|
102
|
+
AND h.rate_observation_started_at_ms IS NOT NULL
|
|
103
|
+
AND GREATEST(h.retry_at_ms, h.cool_until_ms) <= nv.now_ms
|
|
104
|
+
AND nv.now_ms < h.last_ramp_at_ms + ${backpressuredRecoveryInterval}
|
|
105
|
+
THEN h.last_ramp_at_ms + ${backpressuredRecoveryInterval} - nv.now_ms
|
|
106
|
+
ELSE NULL END`
|
|
107
|
+
: 'NULL::bigint';
|
|
108
|
+
// A provider's elapsed hold authorizes one shared recovery probe. Requiring
|
|
109
|
+
// a whole token at the newly learned rate would add another wait when the
|
|
110
|
+
// measured quota cycle is longer than Retry-After itself. The first grant
|
|
111
|
+
// starts an observation epoch, so competing callers cannot reuse this floor.
|
|
112
|
+
const postHoldProbeDue = `h.seen_backpressure
|
|
113
|
+
AND h.rate_observation_started_at_ms IS NULL
|
|
114
|
+
AND h.retry_at_ms > 0
|
|
115
|
+
AND GREATEST(h.retry_at_ms, h.cool_until_ms) <= nv.now_ms`;
|
|
96
116
|
return `provider_hold_rows AS (
|
|
97
117
|
SELECT * FROM ${table} WHERE bucket_id = $1 FOR UPDATE
|
|
98
118
|
), provider_hold AS (
|
|
99
119
|
SELECT h.*,
|
|
100
|
-
|
|
120
|
+
${postHoldProbeDue} AS post_hold_probe_due,
|
|
121
|
+
${pendingRecoveryWaitMs} AS pending_recovery_wait_ms,
|
|
122
|
+
GREATEST(CASE WHEN ${postHoldProbeDue} THEN 1.0 ELSE 0.0 END, LEAST(
|
|
101
123
|
GREATEST(CASE WHEN h.seen_backpressure THEN 1 ELSE h.capacity END, ${refillRate}),
|
|
102
124
|
h.tokens + GREATEST(0, nv.now_ms - h.refilled_at_ms) * ${refillRate} / 1000.0
|
|
103
|
-
) AS tokens_after_refill,
|
|
125
|
+
)) AS tokens_after_refill,
|
|
104
126
|
${observationWasIdle} AS observation_was_idle,
|
|
105
127
|
CASE WHEN ${recoverySuccess}
|
|
106
128
|
AND h.rate_observation_started_at_ms IS NOT NULL
|
|
@@ -140,7 +162,9 @@ export function updateRateScopeSql(
|
|
|
140
162
|
const floorRampedObservation = `${unknownProvider} AND ph.seen_backpressure
|
|
141
163
|
AND ph.refill_per_sec < ${DEFAULT_UNKNOWN_PROVIDER_FAST_RECOVERY_RPS}
|
|
142
164
|
AND ph.next_refill_per_sec > ph.refill_per_sec`;
|
|
143
|
-
const
|
|
165
|
+
const knownRampedObservation = `${!unknownProvider} AND ph.seen_backpressure
|
|
166
|
+
AND ph.next_refill_per_sec > ph.refill_per_sec`;
|
|
167
|
+
const idleFloorObservation = `${unknownProvider} AND ph.observation_was_idle
|
|
144
168
|
AND ph.refill_per_sec < ${DEFAULT_UNKNOWN_PROVIDER_FAST_RECOVERY_RPS}`;
|
|
145
169
|
const rebaseProbe =
|
|
146
170
|
'ph.rate_probe_started_at_ms < ph.last_ramp_at_ms OR ph.rate_probe_started_at_ms < COALESCE(ph.rate_observation_started_at_ms, 0) OR ph.rate_probe_successes_at_ramp > ph.rate_successes';
|
|
@@ -163,28 +187,30 @@ export function updateRateScopeSql(
|
|
|
163
187
|
WHEN (SELECT granted FROM grant_calc) > 0 AND ph.observation_was_idle THEN 0
|
|
164
188
|
ELSE ph.rate_successes END,
|
|
165
189
|
${
|
|
166
|
-
|
|
190
|
+
probeBaselineSupported
|
|
167
191
|
? `rate_probe_successes_at_ramp = CASE
|
|
168
192
|
WHEN ${recoveredObservation} THEN 0
|
|
169
193
|
WHEN (SELECT granted FROM grant_calc) > 0
|
|
170
194
|
AND (ph.rate_observation_started_at_ms IS NULL OR ph.observation_was_idle) THEN 0
|
|
171
195
|
WHEN ${idleFloorObservation} THEN ph.rate_successes
|
|
172
|
-
WHEN ${floorRampedObservation} OR ${probedObservation} OR (${rebaseProbe}) THEN ph.rate_successes
|
|
196
|
+
WHEN ${knownRampedObservation} OR ${floorRampedObservation} OR ${probedObservation} OR (${rebaseProbe}) THEN ph.rate_successes
|
|
173
197
|
ELSE ph.rate_probe_successes_at_ramp END,
|
|
174
198
|
rate_probe_started_at_ms = CASE
|
|
175
|
-
WHEN ${idleFloorObservation} OR ${recoveredObservation} OR ${floorRampedObservation} OR ${probedObservation} OR (${rebaseProbe}) THEN nv.now_ms
|
|
199
|
+
WHEN ${knownRampedObservation} OR ${idleFloorObservation} OR ${recoveredObservation} OR ${floorRampedObservation} OR ${probedObservation} OR (${rebaseProbe}) THEN nv.now_ms
|
|
176
200
|
WHEN (SELECT granted FROM grant_calc) > 0
|
|
177
201
|
AND (ph.rate_observation_started_at_ms IS NULL OR ph.observation_was_idle) THEN nv.now_ms
|
|
178
202
|
ELSE ph.rate_probe_started_at_ms END,`
|
|
179
203
|
: ''
|
|
180
204
|
}
|
|
181
|
-
--
|
|
182
|
-
--
|
|
183
|
-
--
|
|
205
|
+
-- A productive known-provider epoch uses the one-second recovery window,
|
|
206
|
+
-- so its old measurement must not become a failed-probe streak when the
|
|
207
|
+
-- success counter resets. Unknown providers above the floor retain that
|
|
208
|
+
-- measurement for their separate sparse upward-probe policy.
|
|
184
209
|
rate_recovery_interval_ms = CASE
|
|
185
|
-
WHEN
|
|
210
|
+
WHEN (SELECT granted FROM grant_calc) > 0
|
|
186
211
|
AND ph.rate_successes > 0
|
|
187
|
-
AND
|
|
212
|
+
AND (${!unknownProvider}
|
|
213
|
+
OR ph.next_refill_per_sec <= ${DEFAULT_UNKNOWN_PROVIDER_MIN_RPS})
|
|
188
214
|
AND (ph.observation_was_idle
|
|
189
215
|
OR ph.rate_observation_started_at_ms IS NULL)
|
|
190
216
|
-- Zero is an unmeasured new episode, not a failed-probe interval.
|
package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-state.ts
CHANGED
|
@@ -1444,7 +1444,12 @@ async function acquirePureRate(
|
|
|
1444
1444
|
LEAST(
|
|
1445
1445
|
${effectiveCapacitySql('normalized', 'refill_per_sec', unknownProvider)},
|
|
1446
1446
|
ph.tokens_after_refill,
|
|
1447
|
-
|
|
1447
|
+
GREATEST(CASE WHEN ph.post_hold_probe_due
|
|
1448
|
+
AND LEAST(b.tokens, sc.capacity)
|
|
1449
|
+
+ GREATEST(0, nv.now_ms - b.refilled_at_ms) / 1000.0
|
|
1450
|
+
* CASE WHEN sc.adaptive_max_configured
|
|
1451
|
+
THEN sc.max_rps ELSE b.max_rps END >= 1
|
|
1452
|
+
THEN 1.0 ELSE 0.0 END, LEAST(b.tokens, sc.capacity)
|
|
1448
1453
|
+ GREATEST(0, nv.now_ms - b.refilled_at_ms) / 1000.0
|
|
1449
1454
|
* LEAST(
|
|
1450
1455
|
CASE
|
|
@@ -1452,7 +1457,7 @@ async function acquirePureRate(
|
|
|
1452
1457
|
ELSE b.max_rps
|
|
1453
1458
|
END,
|
|
1454
1459
|
GREATEST(sc.min_rps, ph.refill_per_sec)
|
|
1455
|
-
)
|
|
1460
|
+
))
|
|
1456
1461
|
) AS tokens_after_refill
|
|
1457
1462
|
FROM ${bucketTableName(input)} b
|
|
1458
1463
|
JOIN seed_config sc ON sc.rule_id = b.rule_id
|
|
@@ -1683,7 +1688,7 @@ async function acquirePureRate(
|
|
|
1683
1688
|
(SELECT cool_until_ms FROM cooling),
|
|
1684
1689
|
(SELECT retry_at_ms FROM cooling)
|
|
1685
1690
|
) - (SELECT now_ms FROM refreshed LIMIT 1)${fair?.waitBranch ?? ''}
|
|
1686
|
-
ELSE (
|
|
1691
|
+
ELSE LEAST((SELECT pending_recovery_wait_ms FROM provider_hold), (
|
|
1687
1692
|
SELECT CEIL(
|
|
1688
1693
|
MAX(
|
|
1689
1694
|
(
|
|
@@ -1699,7 +1704,7 @@ async function acquirePureRate(
|
|
|
1699
1704
|
)
|
|
1700
1705
|
)::bigint
|
|
1701
1706
|
FROM updated
|
|
1702
|
-
)
|
|
1707
|
+
))
|
|
1703
1708
|
END AS wait_ms
|
|
1704
1709
|
`,
|
|
1705
1710
|
[
|
|
@@ -1964,7 +1969,12 @@ async function grantWithConcurrency(
|
|
|
1964
1969
|
LEAST(
|
|
1965
1970
|
${effectiveCapacitySql('normalized', 'refill_per_sec', unknownProvider)},
|
|
1966
1971
|
ph.tokens_after_refill,
|
|
1967
|
-
|
|
1972
|
+
GREATEST(CASE WHEN ph.post_hold_probe_due
|
|
1973
|
+
AND LEAST(b.tokens, sc.capacity)
|
|
1974
|
+
+ GREATEST(0, nv.now_ms - b.refilled_at_ms) / 1000.0
|
|
1975
|
+
* CASE WHEN sc.adaptive_max_configured
|
|
1976
|
+
THEN sc.max_rps ELSE b.max_rps END >= 1
|
|
1977
|
+
THEN 1.0 ELSE 0.0 END, LEAST(b.tokens, sc.capacity)
|
|
1968
1978
|
+ GREATEST(0, nv.now_ms - b.refilled_at_ms) / 1000.0
|
|
1969
1979
|
* LEAST(
|
|
1970
1980
|
CASE
|
|
@@ -1972,7 +1982,7 @@ async function grantWithConcurrency(
|
|
|
1972
1982
|
ELSE b.max_rps
|
|
1973
1983
|
END,
|
|
1974
1984
|
GREATEST(sc.min_rps, ph.refill_per_sec)
|
|
1975
|
-
)
|
|
1985
|
+
))
|
|
1976
1986
|
) AS tokens_after_refill
|
|
1977
1987
|
FROM ${bucketTableName(input)} b
|
|
1978
1988
|
JOIN seed_config sc ON sc.rule_id = b.rule_id
|
|
@@ -2306,7 +2316,7 @@ async function grantWithConcurrency(
|
|
|
2306
2316
|
(SELECT retry_at_ms FROM cooling)
|
|
2307
2317
|
) - (SELECT now_ms FROM refreshed LIMIT 1)${fair?.waitBranch ?? ''}
|
|
2308
2318
|
WHEN (SELECT free FROM free_slots) <= 0 THEN $7::bigint
|
|
2309
|
-
ELSE (
|
|
2319
|
+
ELSE LEAST((SELECT pending_recovery_wait_ms FROM provider_hold), (
|
|
2310
2320
|
SELECT CEIL(
|
|
2311
2321
|
MAX(
|
|
2312
2322
|
(
|
|
@@ -2323,7 +2333,7 @@ async function grantWithConcurrency(
|
|
|
2323
2333
|
)
|
|
2324
2334
|
)::bigint
|
|
2325
2335
|
FROM updated
|
|
2326
|
-
)
|
|
2336
|
+
))
|
|
2327
2337
|
END AS wait_ms
|
|
2328
2338
|
`,
|
|
2329
2339
|
[
|