deepline 0.3.442 → 0.3.444

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (126) hide show
  1. package/dist/bundling-sources/sdk/src/release.ts +1 -1
  2. package/dist/bundling-sources/sdk/src/types.ts +4 -0
  3. package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +27 -6
  4. package/dist/bundling-sources/shared_libs/play-runtime/gateway-postgres-admission.ts +153 -31
  5. package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts +32 -0
  6. package/dist/bundling-sources/shared_libs/product-notifications/catalog.ts +34 -21
  7. package/dist/bundling-sources/shared_libs/product-notifications/default-policy.ts +1 -1
  8. package/dist/bundling-sources/shared_libs/product-notifications/events.ts +11 -0
  9. package/dist/cli/auth-KWVGTCBU.js +40 -0
  10. package/dist/cli/{auth-CUZJMTNS.mjs → auth-XTFEJXCK.mjs} +4 -4
  11. package/dist/cli/{billing-W2U5VKFI.js → billing-37ATIT7G.js} +73 -73
  12. package/dist/cli/{billing-SWH3WAMC.mjs → billing-3RZZWNAR.mjs} +4 -4
  13. package/dist/cli/{chunk-L3JU24MF.js → chunk-2Y3YBOS4.js} +11 -8
  14. package/dist/cli/{chunk-D7IDBZDR.js → chunk-53HWKCD3.js} +8 -8
  15. package/dist/cli/{chunk-SZUU2WTZ.js → chunk-7J4VZRO5.js} +1 -1
  16. package/dist/cli/{chunk-RWK35AB3.js → chunk-7VAQZERC.js} +6 -6
  17. package/dist/cli/{chunk-L2FWEP4G.mjs → chunk-A5LNESLL.mjs} +4 -4
  18. package/dist/cli/{chunk-7BH4ZOWE.js → chunk-AFFH22ER.js} +2 -2
  19. package/dist/cli/{chunk-P4GMPKMW.mjs → chunk-AJ54W32A.mjs} +1 -1
  20. package/dist/cli/{chunk-KKTRHDLN.mjs → chunk-C4CG252T.mjs} +3 -3
  21. package/dist/cli/{chunk-AR5YQQF5.mjs → chunk-CGVTZKKZ.mjs} +2 -2
  22. package/dist/cli/{chunk-2GNX6SCO.mjs → chunk-CO3S56DX.mjs} +7 -2
  23. package/dist/cli/{chunk-NBFXQCWM.js → chunk-EPK465ZH.js} +8 -8
  24. package/dist/cli/{chunk-KPKU53YO.mjs → chunk-FKAHVFPJ.mjs} +1 -1
  25. package/dist/cli/{chunk-D5IOUCJS.js → chunk-FS7TP7F2.js} +52 -52
  26. package/dist/cli/{chunk-ODWSTSRJ.mjs → chunk-GYLEMRHP.mjs} +5 -5
  27. package/dist/cli/{chunk-6YWIRIOI.mjs → chunk-HPBZAYSS.mjs} +2 -2
  28. package/dist/cli/{chunk-DM6Q634K.js → chunk-HTAXXY66.js} +7 -2
  29. package/dist/cli/{chunk-SKXQZUMX.mjs → chunk-HU6PIQDF.mjs} +2 -2
  30. package/dist/cli/{chunk-5ZQEDZDQ.mjs → chunk-J6NVL526.mjs} +1 -1
  31. package/dist/cli/{chunk-GEHYAQSI.mjs → chunk-JD7GAMDF.mjs} +2 -2
  32. package/dist/cli/{chunk-C4WXHMKN.js → chunk-JMARKXIB.js} +2 -2
  33. package/dist/cli/{chunk-GEQOPXOS.js → chunk-KNAKCMGJ.js} +17 -17
  34. package/dist/cli/{chunk-VUYUI35Q.js → chunk-LRNYRGCN.js} +20 -11
  35. package/dist/cli/{chunk-O765GAOA.mjs → chunk-LS32ZMEI.mjs} +15 -11
  36. package/dist/cli/{chunk-WY4W6PRD.mjs → chunk-LUGZDR4E.mjs} +3 -3
  37. package/dist/cli/{chunk-TP4TMEML.mjs → chunk-MBCQHTXT.mjs} +4 -1
  38. package/dist/cli/{chunk-77DW6WSO.js → chunk-OCYDAMHG.js} +249 -245
  39. package/dist/cli/{chunk-LCT6MTYQ.js → chunk-OTHCIRIC.js} +50 -50
  40. package/dist/cli/{chunk-RR2LZVRA.js → chunk-OZT66JBQ.js} +35 -35
  41. package/dist/cli/{chunk-GXOQRZXG.js → chunk-PP66EATA.js} +2 -2
  42. package/dist/cli/{chunk-R2QWSSAG.js → chunk-QFXPA2E4.js} +7 -7
  43. package/dist/cli/{chunk-SOCWFREM.js → chunk-QR3SSTAE.js} +10 -10
  44. package/dist/cli/{chunk-HYJKJJUM.js → chunk-RNM7JYV6.js} +20 -20
  45. package/dist/cli/{chunk-VLQWLPQU.mjs → chunk-RVM54NYN.mjs} +1 -1
  46. package/dist/cli/{chunk-SMM2EKDC.mjs → chunk-VAQYDKRH.mjs} +1 -1
  47. package/dist/cli/{chunk-7ICMVQL5.mjs → chunk-VHVZVLV7.mjs} +1 -1
  48. package/dist/cli/{chunk-YGMUSZLV.mjs → chunk-VJUFRXFU.mjs} +3 -3
  49. package/dist/cli/{chunk-2JHQ24IT.js → chunk-VVJWB47B.js} +16 -16
  50. package/dist/cli/{chunk-MLJBCS3E.mjs → chunk-W25OALCJ.mjs} +10 -1
  51. package/dist/cli/{chunk-LXTSVUZN.mjs → chunk-WQPJQEDE.mjs} +1 -1
  52. package/dist/cli/{chunk-K2DHO3IM.js → chunk-WYWZZZ2Q.js} +26 -26
  53. package/dist/cli/{chunk-DUUY2JKP.mjs → chunk-X4YO4AXG.mjs} +2 -2
  54. package/dist/cli/{chunk-2GVUZA6C.js → chunk-YIFQGY5A.js} +4 -4
  55. package/dist/cli/command-docs.js +2 -2
  56. package/dist/cli/command-docs.mjs +1 -1
  57. package/dist/cli/{csv-C5Z4OZQN.js → csv-EIJQ6RT5.js} +12 -12
  58. package/dist/cli/{csv-HSR32EPA.mjs → csv-HA2HOVSY.mjs} +3 -3
  59. package/dist/cli/{db-AK4ZYYRP.js → db-V3QK6J3D.js} +25 -25
  60. package/dist/cli/{db-JJGI4J5I.mjs → db-X4YIZUGS.mjs} +4 -4
  61. package/dist/cli/{describe-JVSD6F2G.js → describe-AM3JLWNF.js} +24 -24
  62. package/dist/cli/{describe-C35MTKF4.mjs → describe-MSL3BOWA.mjs} +16 -16
  63. package/dist/cli/describe-core.d.mts +1 -1
  64. package/dist/cli/describe-core.d.ts +1 -1
  65. package/dist/cli/describe-core.js +4 -4
  66. package/dist/cli/describe-core.mjs +3 -3
  67. package/dist/cli/{executions-ZECDB4NA.mjs → executions-EGH4NDY5.mjs} +4 -4
  68. package/dist/cli/{executions-ZEA7O4PX.js → executions-QFQVDHSS.js} +6 -6
  69. package/dist/cli/{feedback-WQC4SX75.js → feedback-77DLEOWO.js} +7 -7
  70. package/dist/cli/{feedback-W7UL4V47.mjs → feedback-RQLOXTWK.mjs} +3 -3
  71. package/dist/cli/git-OIDNWUNY.js +24 -0
  72. package/dist/cli/{git-QBB7LEAV.mjs → git-ZRSKT7ML.mjs} +4 -4
  73. package/dist/cli/index.js +135 -135
  74. package/dist/cli/index.mjs +37 -37
  75. package/dist/cli/{legacy-noops-E5NFHPKQ.mjs → legacy-noops-QNQCSKJV.mjs} +4 -4
  76. package/dist/cli/{legacy-noops-2QPJTEF4.js → legacy-noops-Y3ZMVJSZ.js} +7 -7
  77. package/dist/cli/{monitors-MWBA46VZ.js → monitors-4RY75IKZ.js} +84 -84
  78. package/dist/cli/{monitors-E56RLT6N.mjs → monitors-VEFE3ZSZ.mjs} +4 -4
  79. package/dist/cli/{org-KE2DDK5A.mjs → org-X6ZD3GOS.mjs} +4 -4
  80. package/dist/cli/{org-HGIWXACI.js → org-YOT76AE4.js} +66 -66
  81. package/dist/cli/{play-WO5UXBZM.js → play-7XEA7F7Y.js} +296 -296
  82. package/dist/cli/{play-LGTF4IYG.mjs → play-AXDXVYTK.mjs} +12 -12
  83. package/dist/cli/{providers-AIJPT54M.js → providers-CS3UWBH5.js} +7 -7
  84. package/dist/cli/{providers-77KQS23B.mjs → providers-JKYTNJDM.mjs} +4 -4
  85. package/dist/cli/{quickstart-EFMHLU4T.mjs → quickstart-BHATVOX5.mjs} +3 -3
  86. package/dist/cli/{quickstart-CZQFBJVH.js → quickstart-R2PI3T74.js} +5 -5
  87. package/dist/cli/{search-U5KEPKNV.mjs → search-46CJMKQ7.mjs} +5 -5
  88. package/dist/cli/search-CNDIFIZG.js +30 -0
  89. package/dist/cli/{secrets-JYHZIRO4.mjs → secrets-I2VZ75XW.mjs} +4 -4
  90. package/dist/cli/{secrets-CBBR73U4.js → secrets-NCBNWNNK.js} +13 -13
  91. package/dist/cli/{sessions-XA2EZDXD.js → sessions-SFZPFEXX.js} +5 -5
  92. package/dist/cli/{sessions-T74U74J7.mjs → sessions-UCMJTFYV.mjs} +4 -4
  93. package/dist/cli/{settings-C45BH4CU.mjs → settings-AJ2FOCRS.mjs} +8 -8
  94. package/dist/cli/{settings-ND4437UB.js → settings-N6CUAM55.js} +75 -75
  95. package/dist/cli/{setup-FQ7JS4HM.js → setup-DB34HKIL.js} +42 -42
  96. package/dist/cli/{setup-QDH6FAEZ.mjs → setup-LRWULRUP.mjs} +7 -7
  97. package/dist/cli/skills-Y7CEDG56.js +29 -0
  98. package/dist/cli/{skills-ZFSBA754.mjs → skills-Z5TF33TE.mjs} +5 -5
  99. package/dist/cli/{tools-OVGOFORB.mjs → tools-MOWM5ABI.mjs} +15 -15
  100. package/dist/cli/tools-RQWYUW3K.js +44 -0
  101. package/dist/cli/{update-BNU7VX5M.mjs → update-4LXOHDJU.mjs} +7 -7
  102. package/dist/cli/update-XKRUZQSS.js +31 -0
  103. package/dist/cli/{workflow-FX6GLQ3H.mjs → workflow-IWVOD4N3.mjs} +5 -5
  104. package/dist/cli/{workflow-JSSXWU3T.js → workflow-RS57FL5I.js} +15 -15
  105. package/dist/{client-BqQ7Rb11.d.ts → client-CBitBZpr.d.ts} +15 -3
  106. package/dist/{client-BCmnJaL9.d.mts → client-CTUbBRgg.d.mts} +15 -3
  107. package/dist/index.d.mts +2 -2
  108. package/dist/index.d.ts +2 -2
  109. package/dist/index.js +6 -6
  110. package/dist/index.mjs +3 -3
  111. package/dist/install-integrity.json +90 -90
  112. package/dist/plays/play-description.d.mts +1 -1
  113. package/dist/plays/play-description.d.ts +1 -1
  114. package/dist/plays/run-request.d.mts +1 -1
  115. package/dist/plays/run-request.d.ts +1 -1
  116. package/dist/release.d.mts +1 -1
  117. package/dist/release.d.ts +1 -1
  118. package/dist/release.js +2 -2
  119. package/dist/release.mjs +1 -1
  120. package/package.json +1 -1
  121. package/dist/cli/auth-HAO2GLGH.js +0 -40
  122. package/dist/cli/git-E7CXCBZZ.js +0 -24
  123. package/dist/cli/search-XMNR7CYI.js +0 -30
  124. package/dist/cli/skills-CSA2YFW6.js +0 -29
  125. package/dist/cli/tools-GUALRDI7.js +0 -44
  126. package/dist/cli/update-K4WJSWDB.js +0 -31
@@ -205,7 +205,7 @@ export const SDK_RELEASE = {
205
205
  // getters keep their established compatibility behavior.
206
206
  // 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
207
207
  // 0.3.90 is the first deliberately versioned SDK release for API v3.
208
- version: '0.3.442',
208
+ version: '0.3.444',
209
209
  updateSummary: {
210
210
  introducedIn: '0.3.90',
211
211
  summary:
@@ -445,6 +445,10 @@ export type ToolExecutionMetadata = {
445
445
  windowMs: number;
446
446
  usage: 'queue_hint' | 'enforced' | 'queue_hint_and_enforced' | 'disabled';
447
447
  explicitMaxConcurrency: number | null;
448
+ /** Present when a daily quota restarts at a calendar boundary. */
449
+ resetsAt?: 'utc_midnight';
450
+ /** Present when the rule meters only these credential sources; `env` is Deepline's managed key. */
451
+ credentialSources?: string[];
448
452
  }>;
449
453
  queueHint: {
450
454
  requestsPerSecondHint: number;
@@ -1102,6 +1102,27 @@ function appRuntimeRetryDelayMs(
1102
1102
  return appRuntimeRetryDelaysForAction(action)[attempt - 1] ?? 1_000;
1103
1103
  }
1104
1104
 
1105
+ const APP_RUNTIME_RETRY_AFTER_MIN_SPREAD_MS = 1_000;
1106
+
1107
+ /**
1108
+ * Every caller shed at once gets the same retry-after. Spreading above it,
1109
+ * instead of taking max() with a smaller jittered step, keeps them from
1110
+ * retrying in lockstep.
1111
+ */
1112
+ function appRuntimeRetrySleepMs(
1113
+ ladderDelayMs: number,
1114
+ retryAfterMs: number | null | undefined,
1115
+ ): number {
1116
+ const jitteredMs = applyRetryJitter(ladderDelayMs);
1117
+ const floorMs = retryAfterMs ?? 0;
1118
+ if (floorMs <= jitteredMs) return jitteredMs;
1119
+ const spreadMs = Math.min(
1120
+ floorMs,
1121
+ Math.max(ladderDelayMs, APP_RUNTIME_RETRY_AFTER_MIN_SPREAD_MS),
1122
+ );
1123
+ return floorMs + Math.round(Math.random() * spreadMs);
1124
+ }
1125
+
1105
1126
  function appRuntimeRetryAfterMs(
1106
1127
  response: Response,
1107
1128
  body: string,
@@ -1292,9 +1313,9 @@ async function retryAppRuntimeBodyTimeoutOrThrow(input: {
1292
1313
  error: input.error,
1293
1314
  })
1294
1315
  ) {
1295
- const retryDelayMs = Math.max(
1296
- applyRetryJitter(appRuntimeRetryDelayMs(input.action, input.attempt)),
1297
- input.retryAfterMs ?? 0,
1316
+ const retryDelayMs = appRuntimeRetrySleepMs(
1317
+ appRuntimeRetryDelayMs(input.action, input.attempt),
1318
+ input.retryAfterMs,
1298
1319
  );
1299
1320
  recordAppRuntimeRetryFailure({
1300
1321
  telemetry: input.telemetry,
@@ -1860,9 +1881,9 @@ async function postAppRuntimeApi<TResponse>(
1860
1881
  contentType: response.headers.get('content-type'),
1861
1882
  })
1862
1883
  ) {
1863
- const retryDelayMs = Math.max(
1864
- applyRetryJitter(appRuntimeRetryDelayMs(body.action, attempt)),
1865
- appRuntimeRetryAfterMs(response, responseText) ?? 0,
1884
+ const retryDelayMs = appRuntimeRetrySleepMs(
1885
+ appRuntimeRetryDelayMs(body.action, attempt),
1886
+ appRuntimeRetryAfterMs(response, responseText),
1866
1887
  );
1867
1888
  recordAppRuntimeRetryFailure({
1868
1889
  telemetry: retryTelemetry,
@@ -1,5 +1,9 @@
1
1
  export type GatewayPostgresLane = 'ordinary' | 'commit';
2
2
 
3
+ export type GatewayPostgresAdmissionTenant = string | null;
4
+ export const GATEWAY_POSTGRES_MACHINE_TENANT: GatewayPostgresAdmissionTenant =
5
+ null;
6
+
3
7
  export const GATEWAY_POSTGRES_MAX_ORDINARY_ACTIVE = 8;
4
8
  export const GATEWAY_POSTGRES_MAX_TOTAL_ACTIVE = 10;
5
9
  export const GATEWAY_POSTGRES_MAX_CONSECUTIVE_COMMIT_PRIORITY_GRANTS = 8;
@@ -16,8 +20,10 @@ export const GATEWAY_POSTGRES_MAX_WAITING_PER_LANE = 256;
16
20
  export type GatewayPostgresAdmissionSnapshot = {
17
21
  active: Record<GatewayPostgresLane, number> & { total: number };
18
22
  waiting: Record<GatewayPostgresLane, number>;
23
+ waitingTenants: Record<GatewayPostgresLane, number>;
19
24
  cancelled: number;
20
25
  overloaded: number;
26
+ fairShareEvictions: number;
21
27
  commitPriorityGrants: number;
22
28
  };
23
29
 
@@ -30,7 +36,7 @@ export class GatewayPostgresAdmissionCancelledError extends Error {
30
36
 
31
37
  /**
32
38
  * Load shed at the admission boundary. It is only ever thrown before a waiter
33
- * is enqueued, so no pooled client, statement, or provider call exists for the
39
+ * is granted, so no pooled client, statement, or provider call exists for the
34
40
  * rejected request. Callers must map it to a retryable back-pressure response.
35
41
  */
36
42
  export class GatewayPostgresAdmissionOverloadedError extends Error {
@@ -38,24 +44,31 @@ export class GatewayPostgresAdmissionOverloadedError extends Error {
38
44
  readonly lane: GatewayPostgresLane;
39
45
  readonly waiting: number;
40
46
  readonly limit: number;
47
+ readonly reason: 'queue_full' | 'fair_share_evicted';
41
48
 
42
49
  constructor(input: {
43
50
  lane: GatewayPostgresLane;
44
51
  waiting: number;
45
52
  limit: number;
53
+ reason?: 'queue_full' | 'fair_share_evicted';
46
54
  }) {
55
+ const reason = input.reason ?? 'queue_full';
47
56
  super(
48
- `Gateway Postgres ${input.lane} admission queue is full (${input.waiting} waiting, limit ${input.limit}).`,
57
+ reason === 'queue_full'
58
+ ? `Gateway Postgres ${input.lane} admission queue is full (${input.waiting} waiting, limit ${input.limit}).`
59
+ : `Gateway Postgres ${input.lane} admission queue is full (${input.waiting} waiting, limit ${input.limit}); this waiter yielded its place to an org with a shorter queue.`,
49
60
  );
50
61
  this.name = 'GatewayPostgresAdmissionOverloadedError';
51
62
  this.lane = input.lane;
52
63
  this.waiting = input.waiting;
53
64
  this.limit = input.limit;
65
+ this.reason = reason;
54
66
  }
55
67
  }
56
68
 
57
69
  type Waiter = {
58
70
  lane: GatewayPostgresLane;
71
+ tenant: GatewayPostgresAdmissionTenant;
59
72
  signal: AbortSignal | null;
60
73
  onAbort: (() => void) | null;
61
74
  resolve: (release: () => void) => void;
@@ -63,15 +76,32 @@ type Waiter = {
63
76
  settled: boolean;
64
77
  };
65
78
 
79
+ type LaneQueue = {
80
+ ring: Map<GatewayPostgresAdmissionTenant, Waiter[]>;
81
+ size: number;
82
+ };
83
+
84
+ /**
85
+ * Fair across orgs within each lane, so one org's burst cannot shed or starve
86
+ * every other org on this Machine.
87
+ */
66
88
  export class GatewayPostgresAdmission {
67
- readonly #waiting: Record<GatewayPostgresLane, Waiter[]> = {
68
- ordinary: [],
69
- commit: [],
89
+ readonly #waiting: Record<GatewayPostgresLane, LaneQueue> = {
90
+ ordinary: { ring: new Map(), size: 0 },
91
+ commit: { ring: new Map(), size: 0 },
92
+ };
93
+ readonly #activeByTenant: Record<
94
+ GatewayPostgresLane,
95
+ Map<GatewayPostgresAdmissionTenant, number>
96
+ > = {
97
+ ordinary: new Map(),
98
+ commit: new Map(),
70
99
  };
71
100
  #ordinaryActive = 0;
72
101
  #commitActive = 0;
73
102
  #cancelled = 0;
74
103
  #overloaded = 0;
104
+ #fairShareEvictions = 0;
75
105
  #commitPriorityGrants = 0;
76
106
  #consecutiveCommitPriorityGrants = 0;
77
107
 
@@ -95,29 +125,27 @@ export class GatewayPostgresAdmission {
95
125
 
96
126
  async acquire(
97
127
  lane: GatewayPostgresLane,
98
- options: { signal?: AbortSignal } = {},
128
+ options: {
129
+ signal?: AbortSignal;
130
+ tenant?: GatewayPostgresAdmissionTenant;
131
+ } = {},
99
132
  ): Promise<() => void> {
133
+ const tenant = options.tenant ?? GATEWAY_POSTGRES_MACHINE_TENANT;
100
134
  if (options.signal?.aborted) {
101
135
  this.#cancelled += 1;
102
136
  throw new GatewayPostgresAdmissionCancelledError();
103
137
  }
104
- if (this.#canGrant(lane) && this.#waiting.commit.length === 0) {
105
- return this.#grant(lane);
138
+ if (this.#canGrant(lane) && this.#waiting.commit.size === 0) {
139
+ return this.#grant(lane, tenant);
106
140
  }
107
- const waiting = this.#waiting[lane].length;
108
- const limit = this.#maxWaitingPerLane();
109
- if (waiting >= limit) {
110
- this.#overloaded += 1;
111
- throw new GatewayPostgresAdmissionOverloadedError({
112
- lane,
113
- waiting,
114
- limit,
115
- });
141
+ if (this.#waiting[lane].size >= this.#maxWaitingPerLane()) {
142
+ this.#shedForFullLane(lane, tenant);
116
143
  }
117
144
 
118
145
  return await new Promise<() => void>((resolve, reject) => {
119
146
  const waiter: Waiter = {
120
147
  lane,
148
+ tenant,
121
149
  signal: options.signal ?? null,
122
150
  onAbort: null,
123
151
  resolve,
@@ -130,7 +158,7 @@ export class GatewayPostgresAdmission {
130
158
  reject(new GatewayPostgresAdmissionCancelledError());
131
159
  };
132
160
  waiter.signal?.addEventListener('abort', waiter.onAbort, { once: true });
133
- this.#waiting[lane].push(waiter);
161
+ this.#enqueue(waiter);
134
162
  if (waiter.signal?.aborted) {
135
163
  waiter.onAbort();
136
164
  return;
@@ -147,11 +175,16 @@ export class GatewayPostgresAdmission {
147
175
  total: this.#activeTotal(),
148
176
  },
149
177
  waiting: {
150
- ordinary: this.#waiting.ordinary.length,
151
- commit: this.#waiting.commit.length,
178
+ ordinary: this.#waiting.ordinary.size,
179
+ commit: this.#waiting.commit.size,
180
+ },
181
+ waitingTenants: {
182
+ ordinary: this.#waiting.ordinary.ring.size,
183
+ commit: this.#waiting.commit.ring.size,
152
184
  },
153
185
  cancelled: this.#cancelled,
154
186
  overloaded: this.#overloaded,
187
+ fairShareEvictions: this.#fairShareEvictions,
155
188
  commitPriorityGrants: this.#commitPriorityGrants,
156
189
  };
157
190
  }
@@ -183,15 +216,23 @@ export class GatewayPostgresAdmission {
183
216
  );
184
217
  }
185
218
 
186
- #grant(lane: GatewayPostgresLane): () => void {
219
+ #grant(
220
+ lane: GatewayPostgresLane,
221
+ tenant: GatewayPostgresAdmissionTenant,
222
+ ): () => void {
187
223
  if (lane === 'ordinary') this.#ordinaryActive += 1;
188
224
  else this.#commitActive += 1;
225
+ const activeByTenant = this.#activeByTenant[lane];
226
+ activeByTenant.set(tenant, (activeByTenant.get(tenant) ?? 0) + 1);
189
227
  let released = false;
190
228
  return () => {
191
229
  if (released) return;
192
230
  released = true;
193
231
  if (lane === 'ordinary') this.#ordinaryActive -= 1;
194
232
  else this.#commitActive -= 1;
233
+ const remaining = (activeByTenant.get(tenant) ?? 1) - 1;
234
+ if (remaining > 0) activeByTenant.set(tenant, remaining);
235
+ else activeByTenant.delete(tenant);
195
236
  this.#drain();
196
237
  };
197
238
  }
@@ -199,19 +240,19 @@ export class GatewayPostgresAdmission {
199
240
  #drain(): void {
200
241
  while (this.#activeTotal() < this.#maxTotalActive()) {
201
242
  const ordinaryCanRun =
202
- this.#waiting.ordinary.length > 0 &&
243
+ this.#waiting.ordinary.size > 0 &&
203
244
  this.#ordinaryActive < this.#maxOrdinaryActive();
204
245
  if (
205
246
  ordinaryCanRun &&
206
- this.#waiting.commit.length > 0 &&
247
+ this.#waiting.commit.size > 0 &&
207
248
  this.#consecutiveCommitPriorityGrants >=
208
249
  GATEWAY_POSTGRES_MAX_CONSECUTIVE_COMMIT_PRIORITY_GRANTS
209
250
  ) {
210
251
  this.#consecutiveCommitPriorityGrants = 0;
211
- this.#resolve(this.#waiting.ordinary.shift()!);
252
+ this.#resolve(this.#dequeueLeastActiveTenant('ordinary')!);
212
253
  continue;
213
254
  }
214
- const commit = this.#waiting.commit.shift();
255
+ const commit = this.#dequeueLeastActiveTenant('commit');
215
256
  if (commit) {
216
257
  if (ordinaryCanRun) {
217
258
  this.#commitPriorityGrants += 1;
@@ -224,29 +265,110 @@ export class GatewayPostgresAdmission {
224
265
  }
225
266
  if (ordinaryCanRun) {
226
267
  this.#consecutiveCommitPriorityGrants = 0;
227
- this.#resolve(this.#waiting.ordinary.shift()!);
268
+ this.#resolve(this.#dequeueLeastActiveTenant('ordinary')!);
228
269
  continue;
229
270
  }
230
- if (this.#waiting.ordinary.length === 0) {
271
+ if (this.#waiting.ordinary.size === 0) {
231
272
  this.#consecutiveCommitPriorityGrants = 0;
232
273
  }
233
274
  return;
234
275
  }
235
276
  }
236
277
 
278
+ #enqueue(waiter: Waiter): void {
279
+ const queue = this.#waiting[waiter.lane];
280
+ const tenantQueue = queue.ring.get(waiter.tenant);
281
+ if (tenantQueue) tenantQueue.push(waiter);
282
+ else queue.ring.set(waiter.tenant, [waiter]);
283
+ queue.size += 1;
284
+ }
285
+
286
+ #shedForFullLane(
287
+ lane: GatewayPostgresLane,
288
+ tenant: GatewayPostgresAdmissionTenant,
289
+ ): void {
290
+ const queue = this.#waiting[lane];
291
+ const waiting = queue.size;
292
+ const limit = this.#maxWaitingPerLane();
293
+ const longest = this.#longestTenantQueue(queue);
294
+ const own = queue.ring.get(tenant)?.length ?? 0;
295
+ this.#overloaded += 1;
296
+ if (!longest || own >= longest.length) {
297
+ throw new GatewayPostgresAdmissionOverloadedError({
298
+ lane,
299
+ waiting,
300
+ limit,
301
+ });
302
+ }
303
+ const evicted = longest[longest.length - 1]!;
304
+ this.#remove(evicted);
305
+ this.#fairShareEvictions += 1;
306
+ evicted.reject(
307
+ new GatewayPostgresAdmissionOverloadedError({
308
+ lane,
309
+ waiting,
310
+ limit,
311
+ reason: 'fair_share_evicted',
312
+ }),
313
+ );
314
+ }
315
+
316
+ #dequeueLeastActiveTenant(lane: GatewayPostgresLane): Waiter | undefined {
317
+ const queue = this.#waiting[lane];
318
+ const activeByTenant = this.#activeByTenant[lane];
319
+ let chosen: GatewayPostgresAdmissionTenant | undefined;
320
+ let chosenActive = Infinity;
321
+ for (const tenant of queue.ring.keys()) {
322
+ const active = activeByTenant.get(tenant) ?? 0;
323
+ if (active < chosenActive) {
324
+ chosen = tenant;
325
+ chosenActive = active;
326
+ if (active === 0) break;
327
+ }
328
+ }
329
+ if (chosen === undefined) return undefined;
330
+ const tenantQueue = queue.ring.get(chosen)!;
331
+ const waiter = tenantQueue.shift()!;
332
+ queue.size -= 1;
333
+ this.#rotateToBack(queue, chosen, tenantQueue);
334
+ return waiter;
335
+ }
336
+
337
+ #rotateToBack(
338
+ queue: LaneQueue,
339
+ tenant: GatewayPostgresAdmissionTenant,
340
+ tenantQueue: Waiter[],
341
+ ): void {
342
+ queue.ring.delete(tenant);
343
+ if (tenantQueue.length > 0) queue.ring.set(tenant, tenantQueue);
344
+ }
345
+
346
+ #longestTenantQueue(queue: LaneQueue): Waiter[] | null {
347
+ let longest: Waiter[] | null = null;
348
+ for (const tenantQueue of queue.ring.values()) {
349
+ if (!longest || tenantQueue.length > longest.length) {
350
+ longest = tenantQueue;
351
+ }
352
+ }
353
+ return longest;
354
+ }
355
+
237
356
  #resolve(waiter: Waiter): void {
238
357
  waiter.settled = true;
239
358
  this.#cleanup(waiter);
240
- waiter.resolve(this.#grant(waiter.lane));
359
+ waiter.resolve(this.#grant(waiter.lane, waiter.tenant));
241
360
  }
242
361
 
243
362
  #remove(waiter: Waiter): boolean {
244
363
  if (waiter.settled) return false;
245
364
  const queue = this.#waiting[waiter.lane];
246
- const index = queue.indexOf(waiter);
247
- if (index < 0) return false;
365
+ const tenantQueue = queue.ring.get(waiter.tenant);
366
+ const index = tenantQueue ? tenantQueue.indexOf(waiter) : -1;
367
+ if (!tenantQueue || index < 0) return false;
248
368
  waiter.settled = true;
249
- queue.splice(index, 1);
369
+ tenantQueue.splice(index, 1);
370
+ if (tenantQueue.length === 0) queue.ring.delete(waiter.tenant);
371
+ queue.size -= 1;
250
372
  this.#cleanup(waiter);
251
373
  return true;
252
374
  }
@@ -1,5 +1,7 @@
1
1
  import {
2
2
  acquireRateStateViaAppRuntime,
3
+ AppRuntimeApiResponseError,
4
+ isAppRuntimeApiCapacityError,
3
5
  penalizeRateStateViaAppRuntime,
4
6
  releaseRateStateViaAppRuntime,
5
7
  RateStateValidationUnavailableError,
@@ -141,6 +143,20 @@ function sleepWithAbort(
141
143
  */
142
144
  const RATE_BLOCK_HORIZON_MS = 1_000;
143
145
  const STORE_FAILURE_RETRY_DELAYS_MS = [100, 250] as const;
146
+ const CAPACITY_SHED_DEFAULT_RETRY_AFTER_MS = 1_000;
147
+
148
+ function capacityShedRetryAfterMs(error: unknown): number | null {
149
+ const shed =
150
+ isAppRuntimeApiCapacityError(error) ||
151
+ (error instanceof AppRuntimeApiResponseError &&
152
+ error.retryable &&
153
+ error.status === 429);
154
+ if (!shed) return null;
155
+ const retryAfterMs = (error as AppRuntimeApiResponseError).retryAfterMs;
156
+ return retryAfterMs && retryAfterMs > 0
157
+ ? retryAfterMs
158
+ : CAPACITY_SHED_DEFAULT_RETRY_AFTER_MS;
159
+ }
144
160
  /**
145
161
  * Jitter fraction applied to a server-supplied `waitMs` so a synchronized wave
146
162
  * of buckets that all hit granted=0 at the same instant re-probe at slightly
@@ -1140,6 +1156,22 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
1140
1156
  // A provider hold is not a store failure. Preserve its deadline for
1141
1157
  // the caller's durable suspension or non-durable error path.
1142
1158
  if (error instanceof ProviderExhaustedError) throw error;
1159
+ // A capacity shed is back-pressure from a working store. Wait as for
1160
+ // a zero grant, then return so the acquirer re-checks its wait budget.
1161
+ const shedRetryAfterMs = capacityShedRetryAfterMs(error);
1162
+ if (shedRetryAfterMs !== null) {
1163
+ this.onStoreFailure({
1164
+ bucketId,
1165
+ error: normalizeError(error).message,
1166
+ });
1167
+ await this.sleep(
1168
+ Math.min(
1169
+ shedRetryAfterMs + Math.round(this.jitter() * shedRetryAfterMs),
1170
+ MAX_ACQUIRE_SLEEP_MS,
1171
+ ),
1172
+ );
1173
+ return;
1174
+ }
1143
1175
  // Fail closed. A degraded rate-state store must not grant unlimited
1144
1176
  // provider calls; after this bounded retry ladder the run fails loudly.
1145
1177
  consecutiveFailures += 1;
@@ -21,6 +21,11 @@ import {
21
21
  BILLING_SALES_GROUP_KEY,
22
22
  } from './billing-sales';
23
23
 
24
+ export const DEFAULT_PLAY_FAILURE_SETTINGS = {
25
+ scope: 'automated',
26
+ failureCount: 1,
27
+ } as const satisfies ProductNotificationPlayFailureSettings;
28
+
24
29
  type NotificationTypeDefinition = {
25
30
  id: string;
26
31
  eventName: ProductNotificationEventName;
@@ -244,19 +249,22 @@ function webhookDeliveryPresentation(
244
249
  };
245
250
  }
246
251
 
247
- /** Narrow a Play rule to all Plays, one trigger kind, or a single Play. */
248
- function applyPlayScope(
249
- eventFilter: ProductNotificationFacts,
252
+ /** Translate a Play scope into the facts its events must match. */
253
+ function playScopeFilter(
250
254
  settings: ProductNotificationPlayFailureSettings,
251
- ) {
252
- if (settings.scope === 'cron') eventFilter.triggerKind = 'cron';
253
- if (settings.scope === 'webhook') eventFilter.triggerKind = 'webhook';
255
+ ): ProductNotificationFacts {
256
+ if (settings.scope === 'automated') return { isAutomated: true };
257
+ if (settings.scope === 'cron' || settings.scope === 'webhook') {
258
+ return { triggerKind: settings.scope };
259
+ }
254
260
  if (settings.scope === 'specific') {
255
261
  if (!settings.playId) throw new Error('Choose a specific Play.');
256
- eventFilter.playId = settings.playId;
257
- if (settings.playName) eventFilter.playName = settings.playName;
262
+ return {
263
+ playId: settings.playId,
264
+ ...(settings.playName ? { playName: settings.playName } : {}),
265
+ };
258
266
  }
259
- return eventFilter;
267
+ return {};
260
268
  }
261
269
 
262
270
  function defaultThreshold(
@@ -967,19 +975,22 @@ const PRODUCT_NOTIFICATION_TYPE_REGISTRY = [
967
975
  description:
968
976
  'Notify for all, scheduled, webhook-triggered, or specific Play failures.',
969
977
  allowMultiple: true,
970
- editor: { kind: 'play_failure', defaultFailureCount: 1 },
978
+ editor: {
979
+ kind: 'play_failure',
980
+ defaultFailureCount: DEFAULT_PLAY_FAILURE_SETTINGS.failureCount,
981
+ },
971
982
  },
972
983
  rule: {
984
+ default: {
985
+ ruleKey: 'default.automated_play_failed',
986
+ groupKey: 'default',
987
+ },
973
988
  create: (input, base) => {
974
- const settings = input.playFailure ?? {
975
- scope: 'all' as const,
976
- failureCount: 1,
977
- };
978
- const eventFilter = applyPlayScope({ ...base.eventFilter }, settings);
989
+ const settings = input.playFailure ?? DEFAULT_PLAY_FAILURE_SETTINGS;
979
990
  const count = Math.max(1, Math.floor(settings.failureCount));
980
991
  return {
981
992
  ...base,
982
- eventFilter,
993
+ eventFilter: { ...base.eventFilter, ...playScopeFilter(settings) },
983
994
  repeatMode: count === 1 ? 'every_match' : 'on_enter',
984
995
  trigger:
985
996
  count === 1
@@ -1013,10 +1024,12 @@ const PRODUCT_NOTIFICATION_TYPE_REGISTRY = [
1013
1024
  rule: {
1014
1025
  create: (input, base) => ({
1015
1026
  ...base,
1016
- eventFilter: applyPlayScope(
1017
- { ...base.eventFilter },
1018
- input.playFailure ?? { scope: 'all', failureCount: 1 },
1019
- ),
1027
+ eventFilter: {
1028
+ ...base.eventFilter,
1029
+ ...playScopeFilter(
1030
+ input.playFailure ?? { scope: 'all', failureCount: 1 },
1031
+ ),
1032
+ },
1020
1033
  repeatMode: 'every_match',
1021
1034
  trigger: { kind: 'each_match' },
1022
1035
  }),
@@ -1096,7 +1109,7 @@ export const PRODUCT_NOTIFICATION_TYPE_DEFINITIONS: readonly NotificationTypeDef
1096
1109
  }));
1097
1110
 
1098
1111
  export type ProductNotificationPlayFailureSettings = {
1099
- scope: 'all' | 'cron' | 'webhook' | 'specific';
1112
+ scope: 'all' | 'automated' | 'cron' | 'webhook' | 'specific';
1100
1113
  playId?: string;
1101
1114
  playName?: string;
1102
1115
  failureCount: number;
@@ -20,7 +20,7 @@ import {
20
20
  * they create sparse copy-on-write overrides.
21
21
  */
22
22
  export const CURRENT_PRODUCT_NOTIFICATION_DEFAULT_POLICY = {
23
- version: 11,
23
+ version: 12,
24
24
  groups: [
25
25
  {
26
26
  groupKey: 'default',
@@ -14,6 +14,16 @@ export const PRODUCT_NOTIFICATION_MAX_EVENT_AGE_MS = 30 * 24 * 60 * 60 * 1_000;
14
14
  export const PRODUCT_NOTIFICATION_MAX_IDENTIFIER_LENGTH = 512;
15
15
  export const PRODUCT_NOTIFICATION_MAX_SOURCE_CLOCK_SKEW_MS = 5 * 60 * 1_000;
16
16
 
17
+ export const AUTOMATED_PLAY_TRIGGER_KINDS = ['cron', 'webhook'] as const;
18
+ export type AutomatedPlayTriggerKind =
19
+ (typeof AUTOMATED_PLAY_TRIGGER_KINDS)[number];
20
+
21
+ export function isAutomatedPlayTrigger(
22
+ triggerKind: unknown,
23
+ ): triggerKind is AutomatedPlayTriggerKind {
24
+ return AUTOMATED_PLAY_TRIGGER_KINDS.some((kind) => kind === triggerKind);
25
+ }
26
+
17
27
  export const ProductNotificationEvent = {
18
28
  CreditsBalanceChanged: 'credits.balance.changed',
19
29
  CreditsPurchaseCompleted: 'credits.purchase.completed',
@@ -162,6 +172,7 @@ export const PRODUCT_NOTIFICATION_EVENT_DEFINITIONS: readonly ProductNotificatio
162
172
  },
163
173
  optionalFacts: {
164
174
  triggerKind: 'string',
175
+ isAutomated: 'boolean',
165
176
  },
166
177
  orderingSubjectFact: 'playId',
167
178
  allowedTriggers: ['each_match', 'consecutive_outcomes'],
@@ -0,0 +1,40 @@
1
+ "use strict";Object.defineProperty(exports, "__esModule", {value: true});
2
+
3
+
4
+
5
+
6
+
7
+
8
+
9
+
10
+
11
+
12
+
13
+
14
+
15
+
16
+ var _chunkOTHCIRICjs = require('./chunk-OTHCIRIC.js');
17
+ require('./chunk-2Y3YBOS4.js');
18
+ require('./chunk-YIFQGY5A.js');
19
+ require('./chunk-ALJJRZNJ.js');
20
+ require('./chunk-XNYIZMNQ.js');
21
+ require('./chunk-DIAWYWT5.js');
22
+ require('./chunk-7J4VZRO5.js');
23
+ require('./chunk-OG64RBAC.js');
24
+ require('./chunk-COGTRVWW.js');
25
+
26
+
27
+
28
+
29
+
30
+
31
+
32
+
33
+
34
+
35
+
36
+
37
+
38
+
39
+
40
+ exports.EXIT_WORKSPACE_MISMATCH = _chunkOTHCIRICjs.EXIT_WORKSPACE_MISMATCH; exports.authStatusLine = _chunkOTHCIRICjs.authStatusLine; exports.clearPendingClaim = _chunkOTHCIRICjs.clearPendingClaim; exports.clearSatisfiedPendingClaim = _chunkOTHCIRICjs.clearSatisfiedPendingClaim; exports.handleRegister = _chunkOTHCIRICjs.handleRegister; exports.handleStatus = _chunkOTHCIRICjs.handleStatus; exports.handleWait = _chunkOTHCIRICjs.handleWait; exports.httpJson = _chunkOTHCIRICjs.httpJson; exports.readPendingAuthClaim = _chunkOTHCIRICjs.readPendingAuthClaim; exports.readStatusBanner = _chunkOTHCIRICjs.readStatusBanner; exports.registerAuthCommands = _chunkOTHCIRICjs.registerAuthCommands; exports.reportSetupCompleted = _chunkOTHCIRICjs.reportSetupCompleted; exports.resolvePendingAuthClaimForResume = _chunkOTHCIRICjs.resolvePendingAuthClaimForResume; exports.statusBannerLine = _chunkOTHCIRICjs.statusBannerLine;
@@ -13,13 +13,13 @@ import {
13
13
  reportSetupCompleted,
14
14
  resolvePendingAuthClaimForResume,
15
15
  statusBannerLine
16
- } from "./chunk-AR5YQQF5.mjs";
17
- import "./chunk-TP4TMEML.mjs";
18
- import "./chunk-P4GMPKMW.mjs";
16
+ } from "./chunk-CGVTZKKZ.mjs";
17
+ import "./chunk-MBCQHTXT.mjs";
18
+ import "./chunk-AJ54W32A.mjs";
19
19
  import "./chunk-4QF2K7TM.mjs";
20
20
  import "./chunk-DLIRSHV6.mjs";
21
21
  import "./chunk-HBL4PM5Q.mjs";
22
- import "./chunk-VLQWLPQU.mjs";
22
+ import "./chunk-RVM54NYN.mjs";
23
23
  import "./chunk-WZYKRD7T.mjs";
24
24
  import "./chunk-KRD6H754.mjs";
25
25
  export {