@checkstack/anomaly-backend 1.0.2 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -1,5 +1,221 @@
1
1
  # @checkstack/anomaly-backend
2
2
 
3
+ ## 1.1.0
4
+
5
+ ### Minor Changes
6
+
7
+ - 42abfff: Remove global anomaly settings — configuration is now field-only.
8
+
9
+ `AnomalySettings` (template- and assignment-level) no longer carries
10
+ `sensitivity`, `confirmationWindow`, `driftEnabled`, or `driftThreshold`.
11
+ These were duplicating the per-field configuration path with awkward
12
+ cascade semantics, and a single global multiplier was meaningless across
13
+ fields with different units (ms, %, counts).
14
+
15
+ The schema retains only the truly global concerns:
16
+
17
+ - `enabled` — master kill switch for the assignment
18
+ - `baselineWindow` — there is one history per system, not per field
19
+ - `notify` — one notification preference per assignment
20
+ - `fieldOverrides` — per-field configuration (where everything else now lives)
21
+
22
+ `resolveEffectiveConfig` collapses to two layers: field override → schema
23
+ default → engine fallback constant. The plugin-author defaults set via
24
+ `x-anomaly-*` annotations now drive sensitivity/window/drift across the
25
+ detector and drift evaluator (previously only floors were threaded
26
+ through the schema layer).
27
+
28
+ **Breaking changes:**
29
+
30
+ - Any global `sensitivity`/`confirmationWindow`/`driftEnabled`/
31
+ `driftThreshold` values previously stored in `anomaly_configurations`
32
+ or `anomaly_assignments` are silently stripped on parse. Users who
33
+ customized these globals will revert to the plugin's tuned per-field
34
+ defaults; if they want to keep those values they must re-apply them
35
+ per field in the new UI.
36
+ - `AnomalySettingsForm` no longer renders the global sliders. The form
37
+ now shows: enable toggle, baseline window selector, notify toggle,
38
+ field overrides editor.
39
+ - `AnomalyFieldOverridesEditor` props `defaultSensitivity`,
40
+ `defaultConfirmationWindow`, `defaultDriftEnabled`, `defaultDriftThreshold`
41
+ are removed. Engine fallbacks (1.0, 3, true, 2) are now hard-coded
42
+ internal constants used only when neither field override nor schema
43
+ default is set.
44
+ - The GitOps `System.anomaly` entry schema (in `anomaly-gitops-kinds`)
45
+ drops `sensitivity`, `confirmationWindow`, `driftEnabled`, and
46
+ `driftThreshold` to match the new `AnomalySettings` shape. YAML files
47
+ declaring those fields will be rejected at parse time — operators
48
+ must move per-field tuning into `fieldOverrides`.
49
+
50
+ This change makes the override model trivial to explain ("plugin defaults,
51
+ overridden per field") and removes a class of confusing "where did this
52
+ threshold come from?" questions.
53
+
54
+ - 42abfff: Add practical-significance floors to anomaly detection.
55
+
56
+ Two new schema annotations — `x-anomaly-min-absolute-delta` and `x-anomaly-min-relative-delta` — let plugin authors and operators suppress alerts whose statistical deviation is large but practical impact is negligible. Both floors must clear in addition to the existing μ ± Nσ trigger; defaults are 0 (disabled) so existing behaviour is unchanged.
57
+
58
+ This is the fix for cases like a 6 ms latency baseline whose σ ≈ 1 ms causes routine 20 ms blips to fire as anomalies despite Δ=14 ms being operationally irrelevant. With `min-absolute-delta: 50` and `min-relative-delta: 0.5`, those blips stay silent while a 6 ms → 200 ms spike still fires.
59
+
60
+ Built-in plugins ship with sensible defaults applied to every per-run field: 50 ms + 50 % for ms-unit fields, 5 percentage points for `%`-unit fields, 1 + 25 % for counter fields, 1 GB + 5 % for disk fields, 50 MB + 10 % for memory fields, 1 day for TLS expiry, 0.5 + 25 % for load average, 1 + 5 % for Minecraft TPS. Operators can override per-system or per-field via the assignment UI.
61
+
62
+ - f6f9a5c: Add GitOps extensions for declarative anomaly configuration.
63
+
64
+ Two extensions are now registered against the kind registry:
65
+
66
+ - `Healthcheck.anomaly` — accepts the full `AnomalySettings` shape and
67
+ applies it to the healthcheck's anomaly template via
68
+ `updateAnomalyConfig` on reconcile.
69
+ - `System.anomaly` — accepts an array of per-healthcheck overrides,
70
+ each scoped via `healthcheckRef: { kind: Healthcheck, name: ... }`,
71
+ and applies them with `updateAnomalyAssignmentConfig`. The
72
+ healthcheck reference is the GitOps source of truth; UI edits to
73
+ managed entries are blocked by the existing assignment-level lock.
74
+
75
+ Spec schema documentation for `Healthcheck.anomaly.fieldOverrides` is
76
+ registered **per collector field**, conditioned on the selected
77
+ `collectors[].config` variant — same pattern the `collectors[].assertions`
78
+ docs use, so the kind-registry browser pre-populates the available
79
+ result fields once a collector is chosen. The System extension's
80
+ `fieldOverrides` falls back to a generic variant since the relevant
81
+ collector lives on the referenced Healthcheck rather than a sibling.
82
+
83
+ ### Patch Changes
84
+
85
+ - Updated dependencies [42abfff]
86
+ - Updated dependencies [42abfff]
87
+ - Updated dependencies [f6f9a5c]
88
+ - Updated dependencies [1ef2e79]
89
+ - Updated dependencies [aa89bc5]
90
+ - @checkstack/anomaly-common@1.1.0
91
+ - @checkstack/common@0.9.0
92
+ - @checkstack/gitops-common@0.3.0
93
+ - @checkstack/gitops-backend@0.3.0
94
+ - @checkstack/catalog-common@2.1.0
95
+ - @checkstack/catalog-backend@1.1.0
96
+ - @checkstack/queue-api@0.3.0
97
+ - @checkstack/cache-api@0.3.0
98
+ - @checkstack/backend-api@0.15.1
99
+ - @checkstack/healthcheck-backend@1.0.4
100
+ - @checkstack/healthcheck-common@1.0.2
101
+ - @checkstack/notification-common@1.0.2
102
+ - @checkstack/signal-common@0.2.2
103
+ - @checkstack/cache-utils@0.2.5
104
+
105
+ ## 1.0.3
106
+
107
+ ### Patch Changes
108
+
109
+ - 50e5f5f: Runtime plugin system: install + uninstall plugins from npm, GitHub releases
110
+ (including private GitHub Enterprise instances), or tarball uploads at
111
+ runtime, with multi-package bundles, dependency-derived compatibility checks,
112
+ multi-instance coordination via a Postgres artifact store, and
113
+ single-coordinator destructive cleanup.
114
+
115
+ Highlights:
116
+
117
+ - New `PluginSource` discriminated union and `PluginInstaller` /
118
+ `PluginInstallerRegistry` interfaces in `@checkstack/backend-api`. The
119
+ GitHub variant accepts an optional `apiBaseUrl` so deployments backed by
120
+ GitHub Enterprise can install from `https://ghe.example.com/api/v3`
121
+ instead of `api.github.com`.
122
+ - New `installPackageMetadataSchema` (Zod) in `@checkstack/common` validates
123
+ every plugin's `package.json` at install time. Required fields: `name`,
124
+ `version`, `description`, `author`, `license`, `checkstack.type`,
125
+ `checkstack.pluginId`. Optional: `checkstack.bundle`,
126
+ `checkstack.usageInstructions`, `checkstack.allowInstallScripts`.
127
+ - New `pluginManagerContract` in `@checkstack/pluginmanager-common` with
128
+ `list`, `previewInstall`, `install`, `previewUninstall`, `uninstall`, and
129
+ `events` procedures.
130
+ - New `@checkstack/pluginmanager-frontend` admin UI: installed-plugins list
131
+ with per-row uninstall (typed-confirmation modal, schema/configs/cascade
132
+ toggles), install page with NPM / Tarball Upload / GitHub Release tabs
133
+ (Catalog tab disabled — coming soon), and an events page surfacing the
134
+ install/uninstall audit log.
135
+ - New `bunx @checkstack/scripts plugin-pack` CLI for plugin authors —
136
+ per-package mode produces an npm-shaped tarball; `--bundle` mode produces
137
+ an outer tarball containing every sibling declared in
138
+ `package.json#checkstack.bundle`. Published to npm so external authors
139
+ can `bunx` it directly without a workspace checkout.
140
+ - Compatibility derived from `package.json#dependencies` ranges
141
+ (`semver.satisfies` against the platform's loaded `@checkstack/*`
142
+ versions) — no separate `compatibility` field.
143
+ - Multi-instance: originator persists artifacts + `plugins` rows + broadcasts
144
+ install/uninstall; receiving instances do in-process register/unregister
145
+ only. Destructive ops (drop schema, delete plugin_configs, delete
146
+ artifacts, delete `plugins` rows) run exactly once on the originator.
147
+ - Fresh-instance bootstrap: `loadPlugins()` hydrates any
148
+ `is_uninstallable=true` plugin missing from `node_modules` from the
149
+ artifact store before normal Phase 1 register.
150
+ - New schema: `plugin_artifacts` (tarball storage), `plugin_install_events`
151
+ (audit/error log). `plugins` extended with `version`, `metadata`,
152
+ `source`, `bundle_id`, `is_primary`. Local plugin sync now writes
153
+ `version` from each plugin's `package.json` so the admin UI shows real
154
+ versions instead of `—`.
155
+ - Tarball-upload endpoint (`POST /api/pluginmanager/upload-tarball`) for
156
+ the install UI; access-gated by `pluginmanager.plugin.manage`.
157
+ - Plugin Manager menu link added to the user menu (main grid, alongside
158
+ Profile / Notification Settings / etc.).
159
+
160
+ Cross-cutting changes:
161
+
162
+ - Backend request/response logging now flows through `rootLogger` (winston)
163
+ instead of `hono/logger`. 5xx responses include the response body inline
164
+ so swallowed early-return errors are visible in the log.
165
+ - The `/api/:pluginId/*` dispatcher now logs which core service is missing
166
+ or which `pluginId` had no metadata when it 500s.
167
+ - New `registerCorePluginMetadata` on `PluginManager` for core routers
168
+ (like the plugin manager itself) that need their metadata visible to the
169
+ RPC dispatcher without going through the full plugin lifecycle.
170
+ - ESLint: `unicorn/no-null` is now disabled globally. Drizzle distinguishes
171
+ between `null` (writes a real SQL NULL) and `undefined` (skip the column
172
+ on insert), so treating them as interchangeable produced latent bugs at
173
+ the persistence boundary. The bulk of the patch-bumped packages above
174
+ reflect lint-fix touches that landed when this rule was relaxed.
175
+ - Workspace-wide license normalization to `Elastic-2.0` (matches
176
+ `LICENSE.md`). Every `package.json` in the workspace now declares the
177
+ same SPDX identifier; the patch bumps capture this.
178
+
179
+ Plugin packages (every `plugins/*`): added a `pack` npm script
180
+ (`bunx @checkstack/scripts plugin-pack`), mirrored each plugin's
181
+ `pluginId` from `plugin-metadata.ts` into `package.json#checkstack.pluginId`
182
+ so install-time validation passes, stubbed any missing required metadata
183
+ fields (`description`, `author`, `license`), and added
184
+ `checkstack.bundle` to multi-package plugin primaries (telegram, rcon, ssh,
185
+ jira, queue-bullmq, queue-memory, cache-memory).
186
+
187
+ Breaking changes:
188
+
189
+ - The legacy single-method `PluginInstaller` interface (`install(packageName)`)
190
+ is removed. Callers must use `coreServices.pluginInstallerRegistry`.
191
+ - The old `pluginAdminContract` and `createPluginAdminRouter` are removed.
192
+ Replaced by `pluginManagerContract` in `@checkstack/pluginmanager-common`
193
+ and `createPluginManagerRouter` in `core/backend`.
194
+ - `@checkstack/test-utils-backend` no longer exports
195
+ `createMockPluginInstaller` / `MockPluginInstaller` (the legacy interface
196
+ it shimmed is gone).
197
+
198
+ Note: bumps are limited to `minor` (for packages with new public API
199
+ surface) and `patch` (for downstream consumers, license normalization,
200
+ and lint fixes). No `major` bumps despite the `PluginInstaller` removal —
201
+ the legacy interface had no third-party consumers in the wild before this
202
+ runtime plugin system landed, and the contract surface is the same shape
203
+ modulo the rename.
204
+
205
+ - Updated dependencies [50e5f5f]
206
+ - @checkstack/backend-api@0.15.0
207
+ - @checkstack/catalog-backend@1.0.2
208
+ - @checkstack/catalog-common@2.0.1
209
+ - @checkstack/common@0.8.0
210
+ - @checkstack/healthcheck-backend@1.0.3
211
+ - @checkstack/queue-api@0.2.18
212
+ - @checkstack/anomaly-common@1.0.1
213
+ - @checkstack/cache-api@0.2.4
214
+ - @checkstack/cache-utils@0.2.4
215
+ - @checkstack/healthcheck-common@1.0.1
216
+ - @checkstack/notification-common@1.0.1
217
+ - @checkstack/signal-common@0.2.1
218
+
3
219
  ## 1.0.2
4
220
 
5
221
  ### Patch Changes
package/package.json CHANGED
@@ -1,6 +1,7 @@
1
1
  {
2
2
  "name": "@checkstack/anomaly-backend",
3
- "version": "1.0.2",
3
+ "version": "1.1.0",
4
+ "license": "Elastic-2.0",
4
5
  "type": "module",
5
6
  "main": "src/index.ts",
6
7
  "checkstack": {
@@ -13,28 +14,30 @@
13
14
  "lint:code": "eslint . --max-warnings 0"
14
15
  },
15
16
  "dependencies": {
16
- "@checkstack/backend-api": "0.14.0",
17
- "@checkstack/common": "0.7.0",
18
- "@checkstack/anomaly-common": "1.0.0",
19
- "@checkstack/signal-common": "0.2.0",
20
- "@checkstack/healthcheck-common": "1.0.0",
21
- "@checkstack/queue-api": "0.2.16",
22
- "@checkstack/cache-api": "0.2.2",
23
- "@checkstack/cache-utils": "0.2.2",
24
- "@checkstack/healthcheck-backend": "1.0.1",
25
- "@checkstack/catalog-backend": "1.0.0",
26
- "@checkstack/catalog-common": "2.0.0",
27
- "@checkstack/notification-common": "1.0.0",
17
+ "@checkstack/backend-api": "0.15.0",
18
+ "@checkstack/common": "0.8.0",
19
+ "@checkstack/anomaly-common": "1.0.1",
20
+ "@checkstack/signal-common": "0.2.1",
21
+ "@checkstack/healthcheck-common": "1.0.1",
22
+ "@checkstack/queue-api": "0.2.18",
23
+ "@checkstack/cache-api": "0.2.4",
24
+ "@checkstack/cache-utils": "0.2.4",
25
+ "@checkstack/healthcheck-backend": "1.0.3",
26
+ "@checkstack/catalog-backend": "1.0.2",
27
+ "@checkstack/gitops-backend": "0.2.8",
28
+ "@checkstack/gitops-common": "0.2.2",
29
+ "@checkstack/catalog-common": "2.0.1",
30
+ "@checkstack/notification-common": "1.0.1",
28
31
  "drizzle-orm": "^0.45.0",
29
32
  "hono": "^4.12.14",
30
33
  "zod": "^4.2.1",
31
34
  "@orpc/server": "^1.13.2"
32
35
  },
33
36
  "devDependencies": {
34
- "@checkstack/drizzle-helper": "0.0.4",
35
- "@checkstack/scripts": "0.1.2",
36
- "@checkstack/test-utils-backend": "0.1.22",
37
- "@checkstack/tsconfig": "0.0.5",
37
+ "@checkstack/drizzle-helper": "0.0.5",
38
+ "@checkstack/scripts": "0.3.0",
39
+ "@checkstack/test-utils-backend": "0.1.24",
40
+ "@checkstack/tsconfig": "0.0.7",
38
41
  "@types/bun": "^1.0.0",
39
42
  "date-fns": "^4.1.0",
40
43
  "drizzle-kit": "^0.31.10",
@@ -0,0 +1,189 @@
1
+ import { describe, it, expect, mock, beforeEach } from "bun:test";
2
+ import type { ReconcileContext } from "@checkstack/gitops-common";
3
+ import type { AnomalyService } from "./service";
4
+ import {
5
+ buildHealthcheckAnomalyExtension,
6
+ buildSystemAnomalyExtension,
7
+ } from "./anomaly-gitops-kinds";
8
+
9
+ /**
10
+ * Reconcile-time tests for the anomaly-backend GitOps kind extensions.
11
+ *
12
+ * Exercises the `Healthcheck.anomaly` and `System.anomalyExceptions`
13
+ * extensions in isolation against a stub AnomalyService.
14
+ */
15
+
16
+ const noopLogger = {
17
+ debug: () => {},
18
+ info: () => {},
19
+ warn: () => {},
20
+ error: () => {},
21
+ };
22
+
23
+ const buildContext = (
24
+ overrides: Partial<ReconcileContext> = {},
25
+ ): ReconcileContext =>
26
+ ({
27
+ logger: noopLogger,
28
+ resolveEntityRef: async () => undefined,
29
+ resolveSecretsBySchema: async ({ value }) => ({
30
+ resolved: value,
31
+ warnings: [],
32
+ }),
33
+ ...overrides,
34
+ }) as ReconcileContext;
35
+
36
+ const stubService = () => {
37
+ const updateAnomalyConfig = mock(async () => ({}));
38
+ const updateAnomalyAssignmentConfig = mock(async () => ({}));
39
+ return {
40
+ updateAnomalyConfig,
41
+ updateAnomalyAssignmentConfig,
42
+ service: {
43
+ updateAnomalyConfig,
44
+ updateAnomalyAssignmentConfig,
45
+ } as unknown as AnomalyService,
46
+ };
47
+ };
48
+
49
+ describe("buildHealthcheckAnomalyExtension", () => {
50
+ let stub: ReturnType<typeof stubService>;
51
+ let extension: ReturnType<typeof buildHealthcheckAnomalyExtension>;
52
+
53
+ beforeEach(() => {
54
+ stub = stubService();
55
+ extension = buildHealthcheckAnomalyExtension({
56
+ getService: () => stub.service,
57
+ });
58
+ });
59
+
60
+ it("registers under the right kind/namespace", () => {
61
+ expect(extension.kind).toBe("Healthcheck");
62
+ expect(extension.namespace).toBe("anomaly");
63
+ });
64
+
65
+ it("calls updateAnomalyConfig with the spec when present", async () => {
66
+ await extension.reconcile({
67
+ entity: {} as never,
68
+ extensionSpec: {
69
+ enabled: false,
70
+ baselineWindow: "14d",
71
+ notify: false,
72
+ },
73
+ entityId: "hc-1",
74
+ context: buildContext(),
75
+ });
76
+ expect(stub.updateAnomalyConfig).toHaveBeenCalledTimes(1);
77
+ expect(stub.updateAnomalyConfig).toHaveBeenCalledWith("hc-1", {
78
+ enabled: false,
79
+ baselineWindow: "14d",
80
+ notify: false,
81
+ });
82
+ });
83
+
84
+ it("is a no-op when extensionSpec is undefined", async () => {
85
+ await extension.reconcile({
86
+ entity: {} as never,
87
+ extensionSpec: undefined,
88
+ entityId: "hc-1",
89
+ context: buildContext(),
90
+ });
91
+ expect(stub.updateAnomalyConfig).not.toHaveBeenCalled();
92
+ });
93
+ });
94
+
95
+ describe("buildSystemAnomalyExtension", () => {
96
+ let stub: ReturnType<typeof stubService>;
97
+ let extension: ReturnType<typeof buildSystemAnomalyExtension>;
98
+
99
+ beforeEach(() => {
100
+ stub = stubService();
101
+ extension = buildSystemAnomalyExtension({
102
+ getService: () => stub.service,
103
+ });
104
+ });
105
+
106
+ it("registers on the System kind under namespace 'anomaly'", () => {
107
+ expect(extension.kind).toBe("System");
108
+ expect(extension.namespace).toBe("anomaly");
109
+ });
110
+
111
+ it("resolves each healthcheckRef and writes assignment overrides", async () => {
112
+ const refMap: Record<string, string> = {
113
+ "hc-cpu": "cfg-cpu",
114
+ "hc-mem": "cfg-mem",
115
+ };
116
+
117
+ await extension.reconcile({
118
+ entity: {} as never,
119
+ extensionSpec: [
120
+ {
121
+ healthcheckRef: { kind: "Healthcheck", name: "hc-cpu" },
122
+ enabled: false,
123
+ fieldOverrides: {
124
+ "cpu.usage": { sensitivity: 0.5 },
125
+ },
126
+ },
127
+ {
128
+ healthcheckRef: { kind: "Healthcheck", name: "hc-mem" },
129
+ baselineWindow: "30d",
130
+ },
131
+ ],
132
+ entityId: "sys-1",
133
+ context: buildContext({
134
+ resolveEntityRef: async ({ entityName }) => refMap[entityName],
135
+ }),
136
+ });
137
+
138
+ expect(stub.updateAnomalyAssignmentConfig).toHaveBeenCalledTimes(2);
139
+ expect(stub.updateAnomalyAssignmentConfig).toHaveBeenNthCalledWith(
140
+ 1,
141
+ "sys-1",
142
+ "cfg-cpu",
143
+ {
144
+ enabled: false,
145
+ fieldOverrides: { "cpu.usage": { sensitivity: 0.5 } },
146
+ },
147
+ );
148
+ expect(stub.updateAnomalyAssignmentConfig).toHaveBeenNthCalledWith(
149
+ 2,
150
+ "sys-1",
151
+ "cfg-mem",
152
+ { baselineWindow: "30d" },
153
+ );
154
+ });
155
+
156
+ it("throws when a healthcheck ref cannot be resolved", async () => {
157
+ await expect(
158
+ extension.reconcile({
159
+ entity: {} as never,
160
+ extensionSpec: [
161
+ {
162
+ healthcheckRef: { kind: "Healthcheck", name: "missing" },
163
+ enabled: false,
164
+ },
165
+ ],
166
+ entityId: "sys-1",
167
+ context: buildContext({
168
+ resolveEntityRef: async () => undefined,
169
+ }),
170
+ }),
171
+ ).rejects.toThrow(/Cannot resolve Healthcheck ref "missing"/);
172
+ });
173
+
174
+ it("is a no-op for empty/undefined specs", async () => {
175
+ await extension.reconcile({
176
+ entity: {} as never,
177
+ extensionSpec: [],
178
+ entityId: "sys-1",
179
+ context: buildContext(),
180
+ });
181
+ await extension.reconcile({
182
+ entity: {} as never,
183
+ extensionSpec: undefined,
184
+ entityId: "sys-1",
185
+ context: buildContext(),
186
+ });
187
+ expect(stub.updateAnomalyAssignmentConfig).not.toHaveBeenCalled();
188
+ });
189
+ });
@@ -0,0 +1,234 @@
1
+ import { z } from "zod";
2
+ import {
3
+ CHECKSTACK_API_VERSION,
4
+ type EntityKindExtensionDefinition,
5
+ type EntityKindRegistry,
6
+ type ReconcileContext,
7
+ } from "@checkstack/gitops-common";
8
+ import {
9
+ AnomalySettingsSchema,
10
+ AnomalyFieldConfigSchema,
11
+ } from "@checkstack/anomaly-common";
12
+ import type { CollectorRegistry } from "@checkstack/backend-api";
13
+ import type { AnomalyService } from "./service";
14
+
15
+ interface AnomalyGitOpsKindsDeps {
16
+ /**
17
+ * Lazy accessor — populated during init(), invoked at reconcile time.
18
+ * Mirrors the pattern healthcheck-backend uses to expose its service to
19
+ * the GitOps reconciler without bleeding init order into kind registration.
20
+ */
21
+ getService: () => AnomalyService;
22
+ }
23
+
24
+ // ─── Shared schemas ─────────────────────────────────────────────────────────
25
+
26
+ /**
27
+ * A reference to a Healthcheck entity. Constrained so YAML callers can't
28
+ * accidentally point an anomaly override at a non-Healthcheck kind.
29
+ */
30
+ const healthcheckRefSchema = z
31
+ .object({
32
+ kind: z.literal("Healthcheck"),
33
+ name: z.string().min(1),
34
+ })
35
+ .describe("Reference to the Healthcheck this anomaly override applies to.");
36
+
37
+ // ─── Healthcheck → anomaly extension ────────────────────────────────────────
38
+ //
39
+ // Declares the *template-level* AnomalySettings for a Healthcheck. GitOps is
40
+ // the source of truth, so the entire AnomalySettings record is replaced.
41
+
42
+ const healthcheckAnomalyExtensionSchema = AnomalySettingsSchema.optional();
43
+ type HealthcheckAnomalyExtension = z.infer<
44
+ typeof healthcheckAnomalyExtensionSchema
45
+ >;
46
+
47
+ export function buildHealthcheckAnomalyExtension(
48
+ deps: AnomalyGitOpsKindsDeps,
49
+ ): EntityKindExtensionDefinition<HealthcheckAnomalyExtension> {
50
+ return {
51
+ apiVersion: CHECKSTACK_API_VERSION,
52
+ kind: "Healthcheck",
53
+ namespace: "anomaly",
54
+ specSchema: healthcheckAnomalyExtensionSchema,
55
+
56
+ reconcile: async ({
57
+ extensionSpec,
58
+ entityId,
59
+ context,
60
+ }: {
61
+ extensionSpec: HealthcheckAnomalyExtension;
62
+ entityId: string;
63
+ context: ReconcileContext;
64
+ }) => {
65
+ if (!extensionSpec) return;
66
+ await deps.getService().updateAnomalyConfig(entityId, extensionSpec);
67
+ context.logger.info(
68
+ `GitOps: applied anomaly defaults to Healthcheck (id: ${entityId})`,
69
+ );
70
+ },
71
+ };
72
+ }
73
+
74
+ // ─── System → anomaly extension ─────────────────────────────────────────────
75
+ //
76
+ // Declares per-assignment AnomalySettings overrides ("exceptions"), keyed by
77
+ // healthcheck ref. Lives under namespace `anomaly` on the System kind so the
78
+ // YAML naming mirrors the Healthcheck.anomaly extension.
79
+
80
+ const systemAnomalyEntrySchema = z.object({
81
+ healthcheckRef: healthcheckRefSchema,
82
+ enabled: z.boolean().optional(),
83
+ baselineWindow: z.string().optional(),
84
+ notify: z.boolean().optional(),
85
+ fieldOverrides: z.record(z.string(), AnomalyFieldConfigSchema).optional(),
86
+ });
87
+
88
+ const systemAnomalyExtensionSchema = z.array(systemAnomalyEntrySchema).optional();
89
+
90
+ type SystemAnomalyExtension = z.infer<typeof systemAnomalyExtensionSchema>;
91
+
92
+ export function buildSystemAnomalyExtension(
93
+ deps: AnomalyGitOpsKindsDeps,
94
+ ): EntityKindExtensionDefinition<SystemAnomalyExtension> {
95
+ return {
96
+ apiVersion: CHECKSTACK_API_VERSION,
97
+ kind: "System",
98
+ namespace: "anomaly",
99
+ specSchema: systemAnomalyExtensionSchema,
100
+
101
+ reconcile: async ({
102
+ extensionSpec,
103
+ entityId,
104
+ context,
105
+ }: {
106
+ extensionSpec: SystemAnomalyExtension;
107
+ entityId: string;
108
+ context: ReconcileContext;
109
+ }) => {
110
+ if (!extensionSpec || extensionSpec.length === 0) return;
111
+
112
+ const service = deps.getService();
113
+ const systemId = entityId;
114
+
115
+ for (const entry of extensionSpec) {
116
+ const configurationId = await context.resolveEntityRef({
117
+ kind: entry.healthcheckRef.kind,
118
+ entityName: entry.healthcheckRef.name,
119
+ });
120
+ if (!configurationId) {
121
+ throw new Error(
122
+ `Cannot resolve Healthcheck ref "${entry.healthcheckRef.name}" — anomaly overrides require the healthcheck to exist`,
123
+ );
124
+ }
125
+
126
+ const { healthcheckRef: _ref, ...overrides } = entry;
127
+ void _ref;
128
+
129
+ await service.updateAnomalyAssignmentConfig(
130
+ systemId,
131
+ configurationId,
132
+ overrides,
133
+ );
134
+
135
+ context.logger.info(
136
+ `GitOps: applied anomaly overrides for Healthcheck "${entry.healthcheckRef.name}" on System (id: ${systemId})`,
137
+ );
138
+ }
139
+ },
140
+ };
141
+ }
142
+
143
+ // ─── Documentation ──────────────────────────────────────────────────────────
144
+
145
+ /**
146
+ * Mirrors the unwrap helper in healthcheck-gitops-kinds — peels Optional /
147
+ * Nullable / Default wrappers so we can introspect a collector's result
148
+ * shape and enumerate its field names.
149
+ */
150
+ function unwrapZodType(type: z.ZodTypeAny): z.ZodTypeAny {
151
+ let current = type;
152
+ while (current) {
153
+ if (current instanceof z.ZodOptional || current instanceof z.ZodNullable) {
154
+ current = current.unwrap() as z.ZodTypeAny;
155
+ } else if (current instanceof z.ZodDefault) {
156
+ current = current._def.innerType as z.ZodTypeAny;
157
+ } else if ("innerType" in current && typeof current.innerType === "function") {
158
+ current = current.innerType() as z.ZodTypeAny;
159
+ } else {
160
+ break;
161
+ }
162
+ }
163
+ return current;
164
+ }
165
+
166
+ /**
167
+ * Register schema documentation for the anomaly extensions. Called from
168
+ * anomaly-backend's `afterPluginsReady` once the kind registry and the
169
+ * collector registry have been populated.
170
+ *
171
+ * The Healthcheck.anomaly.fieldOverrides documentation is registered
172
+ * **per collector field**: each variant is gated on the matching
173
+ * collectors[].config selection so the kind-registry browser can pre-populate
174
+ * the available result fields once the user picks a collector.
175
+ *
176
+ * The System.anomaly[].fieldOverrides path can't be conditioned on a sibling
177
+ * config (the collector lives on the referenced Healthcheck, not on the
178
+ * System spec), so it falls back to a generic AnomalyFieldConfig variant.
179
+ */
180
+ export function registerAnomalyGitOpsDocumentation({
181
+ kindRegistry,
182
+ collectorRegistry,
183
+ }: {
184
+ kindRegistry: EntityKindRegistry;
185
+ collectorRegistry: CollectorRegistry;
186
+ }): void {
187
+ // Per-collector, per-field variants conditioned on the chosen collector.
188
+ for (const registered of collectorRegistry.getCollectors()) {
189
+ const unwrapped = unwrapZodType(registered.collector.result.schema);
190
+ if (!(unwrapped instanceof z.ZodObject)) continue;
191
+
192
+ for (const fieldName of Object.keys(unwrapped.shape)) {
193
+ kindRegistry.registerSpecSchemaDocumentation({
194
+ apiVersion: CHECKSTACK_API_VERSION,
195
+ kind: "Healthcheck",
196
+ fieldPath: "anomaly.fieldOverrides",
197
+ variantId: `${registered.qualifiedId}.field.${fieldName}`,
198
+ label: `Field: ${fieldName}`,
199
+ description: `Anomaly engine override for the "${fieldName}" result field of ${registered.collector.displayName}. The map key is "${fieldName}".`,
200
+ schema: AnomalyFieldConfigSchema,
201
+ conditions: [
202
+ {
203
+ fieldPath: "collectors[].config",
204
+ variantIds: [registered.qualifiedId],
205
+ },
206
+ ],
207
+ });
208
+ }
209
+ }
210
+
211
+ // Generic fallback for the System extension — the relevant collector lives
212
+ // on the referenced Healthcheck, so we can't gate on a sibling.
213
+ kindRegistry.registerSpecSchemaDocumentation({
214
+ apiVersion: CHECKSTACK_API_VERSION,
215
+ kind: "System",
216
+ fieldPath: "anomaly[].fieldOverrides",
217
+ label: "Anomaly field override",
218
+ description:
219
+ "Per-result-field overrides applied to the System ↔ Healthcheck assignment. The map key is the field path of the referenced healthcheck's result (e.g. \"cpu.usage\", \"latencyMs\").",
220
+ schema: AnomalyFieldConfigSchema,
221
+ });
222
+ }
223
+
224
+ // ─── Registration entry point ───────────────────────────────────────────────
225
+
226
+ export function registerAnomalyGitOpsKinds({
227
+ kindRegistry,
228
+ ...deps
229
+ }: AnomalyGitOpsKindsDeps & {
230
+ kindRegistry: EntityKindRegistry;
231
+ }): void {
232
+ kindRegistry.registerKindExtension(buildHealthcheckAnomalyExtension(deps));
233
+ kindRegistry.registerKindExtension(buildSystemAnomalyExtension(deps));
234
+ }
@@ -516,7 +516,7 @@ describe("Anomaly Detector — processCheckCompleted", () => {
516
516
  const db = createMockDb({
517
517
  configRecord: {
518
518
  version: 1,
519
- data: { enabled: false, sensitivity: 1, confirmationWindow: 3, baselineWindow: "7d", notify: true, driftEnabled: true, driftThreshold: 2 } satisfies AnomalySettings,
519
+ data: { enabled: false, baselineWindow: "7d", notify: true } satisfies AnomalySettings,
520
520
  },
521
521
  });
522
522
 
package/src/detector.ts CHANGED
@@ -157,18 +157,13 @@ export async function processCheckCompleted({
157
157
  continue; // Learning phase (no baseline yet)
158
158
  }
159
159
 
160
- const {
161
- enabled: effectiveEnabled,
162
- sensitivity: effectiveSensitivity,
163
- confirmationWindow: effectiveConfirmation,
164
- direction: effectiveDirection,
165
- } = resolveEffectiveConfig(path, templateConfig, assignmentConfig);
166
-
167
- if (!effectiveEnabled) {
168
- continue;
169
- }
170
-
171
160
  let schemaDirection: AnomalyDirection | undefined;
161
+ let schemaSensitivity: number | undefined;
162
+ let schemaConfirmationWindow: number | undefined;
163
+ let schemaDriftEnabled: boolean | undefined;
164
+ let schemaDriftThreshold: number | undefined;
165
+ let schemaMinAbsoluteDelta: number | undefined;
166
+ let schemaMinRelativeDelta: number | undefined;
172
167
  const collector = collectorRegistry.getCollector(collectorId);
173
168
  if (collector) {
174
169
  const collectorSchema = collector.collector.result.schema;
@@ -178,10 +173,36 @@ export async function processCheckCompleted({
178
173
  if (fieldSchema) {
179
174
  const meta = getHealthResultMeta(fieldSchema);
180
175
  schemaDirection = meta?.["x-anomaly-direction"];
176
+ schemaSensitivity = meta?.["x-anomaly-sensitivity"];
177
+ schemaConfirmationWindow = meta?.["x-anomaly-confirmation-window"];
178
+ schemaDriftEnabled = meta?.["x-anomaly-drift-enabled"];
179
+ schemaDriftThreshold = meta?.["x-anomaly-drift-threshold"];
180
+ schemaMinAbsoluteDelta = meta?.["x-anomaly-min-absolute-delta"];
181
+ schemaMinRelativeDelta = meta?.["x-anomaly-min-relative-delta"];
181
182
  }
182
183
  }
183
184
  }
184
185
 
186
+ const {
187
+ enabled: effectiveEnabled,
188
+ sensitivity: effectiveSensitivity,
189
+ confirmationWindow: effectiveConfirmation,
190
+ direction: effectiveDirection,
191
+ minAbsoluteDelta: effectiveMinAbsolute,
192
+ minRelativeDelta: effectiveMinRelative,
193
+ } = resolveEffectiveConfig(path, templateConfig, assignmentConfig, {
194
+ sensitivity: schemaSensitivity,
195
+ confirmationWindow: schemaConfirmationWindow,
196
+ driftEnabled: schemaDriftEnabled,
197
+ driftThreshold: schemaDriftThreshold,
198
+ minAbsoluteDelta: schemaMinAbsoluteDelta,
199
+ minRelativeDelta: schemaMinRelativeDelta,
200
+ });
201
+
202
+ if (!effectiveEnabled) {
203
+ continue;
204
+ }
205
+
185
206
  const direction = effectiveDirection ?? schemaDirection;
186
207
 
187
208
  if (!direction) {
@@ -206,7 +227,13 @@ export async function processCheckCompleted({
206
227
  direction,
207
228
  effectiveSensitivity,
208
229
  );
209
- anomalous = isAnomalous(value, thresholds);
230
+ anomalous = isAnomalous({
231
+ value,
232
+ mean: baseline.mean,
233
+ thresholds,
234
+ minAbsoluteDelta: effectiveMinAbsolute,
235
+ minRelativeDelta: effectiveMinRelative,
236
+ });
210
237
  deviation =
211
238
  baseline.stdDev > 0
212
239
  ? Math.abs(value - baseline.mean) / baseline.stdDev
@@ -121,12 +121,8 @@ const stableBaseline = createBaseline({
121
121
 
122
122
  const defaultTemplate: AnomalySettings = {
123
123
  enabled: true,
124
- sensitivity: 1,
125
- confirmationWindow: 3,
126
124
  baselineWindow: "7d",
127
125
  notify: true,
128
- driftEnabled: true,
129
- driftThreshold: 2,
130
126
  };
131
127
 
132
128
  describe("evaluateDrift", () => {
@@ -162,16 +158,21 @@ describe("evaluateDrift", () => {
162
158
  expect(db._insertCalls.length).toBe(0);
163
159
  });
164
160
 
165
- test("does nothing when driftEnabled is false", async () => {
161
+ test("does nothing when drift is disabled at the field level", async () => {
166
162
  const db = createMockDb();
167
163
  await evaluateDrift({
168
164
  ...baseProps,
169
165
  baseline: driftingBaseline,
170
166
  schemaDirection: "lower-is-better",
171
- templateConfig: { ...defaultTemplate, driftEnabled: false },
167
+ templateConfig: {
168
+ ...defaultTemplate,
169
+ fieldOverrides: {
170
+ "collectors.http.request.responseTimeMs": { driftEnabled: false },
171
+ },
172
+ },
172
173
  db: db as never,
173
174
  catalogClient: createMockCatalogClient() as never,
174
- notificationClient: createMockNotificationClient() as never,
175
+ notificationClient: createMockNotificationClient() as never,
175
176
  logger: createMockLogger() as never,
176
177
  });
177
178
  expect(db._insertCalls.length).toBe(0);
@@ -33,6 +33,18 @@ export interface EvaluateDriftInput {
33
33
  baseline: FieldBaseline;
34
34
  /** Direction declared by the schema for this field, if any. */
35
35
  schemaDirection?: AnomalyDirection;
36
+ /** Schema-declared sensitivity multiplier (plugin author default). */
37
+ schemaSensitivity?: number;
38
+ /** Schema-declared confirmation window (plugin author default). */
39
+ schemaConfirmationWindow?: number;
40
+ /** Schema-declared drift toggle (plugin author default). */
41
+ schemaDriftEnabled?: boolean;
42
+ /** Schema-declared drift threshold sigma multiplier (plugin author default). */
43
+ schemaDriftThreshold?: number;
44
+ /** Schema-declared practical-significance floor on absolute change. */
45
+ schemaMinAbsoluteDelta?: number;
46
+ /** Schema-declared practical-significance floor on relative change. */
47
+ schemaMinRelativeDelta?: number;
36
48
  templateConfig?: AnomalySettings;
37
49
  assignmentConfig?: Partial<AnomalySettings>;
38
50
  }
@@ -58,6 +70,12 @@ export async function evaluateDrift({
58
70
  fieldPath,
59
71
  baseline,
60
72
  schemaDirection,
73
+ schemaSensitivity,
74
+ schemaConfirmationWindow,
75
+ schemaDriftEnabled,
76
+ schemaDriftThreshold,
77
+ schemaMinAbsoluteDelta,
78
+ schemaMinRelativeDelta,
61
79
  templateConfig,
62
80
  assignmentConfig,
63
81
  }: EvaluateDriftInput): Promise<void> {
@@ -67,7 +85,16 @@ export async function evaluateDrift({
67
85
  direction: configDirection,
68
86
  driftEnabled,
69
87
  driftThreshold,
70
- } = resolveEffectiveConfig(fieldPath, templateConfig, assignmentConfig);
88
+ minAbsoluteDelta,
89
+ minRelativeDelta,
90
+ } = resolveEffectiveConfig(fieldPath, templateConfig, assignmentConfig, {
91
+ sensitivity: schemaSensitivity,
92
+ confirmationWindow: schemaConfirmationWindow,
93
+ driftEnabled: schemaDriftEnabled,
94
+ driftThreshold: schemaDriftThreshold,
95
+ minAbsoluteDelta: schemaMinAbsoluteDelta,
96
+ minRelativeDelta: schemaMinRelativeDelta,
97
+ });
71
98
 
72
99
  const direction = configDirection ?? schemaDirection;
73
100
 
@@ -83,6 +110,9 @@ export async function evaluateDrift({
83
110
  direction,
84
111
  sensitivity,
85
112
  threshold: driftThreshold,
113
+ mean: baseline.mean,
114
+ minAbsoluteDelta,
115
+ minRelativeDelta,
86
116
  });
87
117
 
88
118
  const [existing] = await db
@@ -205,7 +205,7 @@ export async function setupBaselineAnalyzerJob({
205
205
  const fieldName = fieldNames[path];
206
206
  if (!collectorId || !fieldName) continue;
207
207
 
208
- const schemaDirection = lookupSchemaDirection({
208
+ const schemaInfo = lookupSchemaInfo({
209
209
  collectorRegistry,
210
210
  collectorId,
211
211
  fieldName,
@@ -226,7 +226,13 @@ export async function setupBaselineAnalyzerJob({
226
226
  configurationId: assignment.configurationId,
227
227
  fieldPath: path,
228
228
  baseline: baselineDto,
229
- schemaDirection,
229
+ schemaDirection: schemaInfo.direction,
230
+ schemaSensitivity: schemaInfo.sensitivity,
231
+ schemaConfirmationWindow: schemaInfo.confirmationWindow,
232
+ schemaDriftEnabled: schemaInfo.driftEnabled,
233
+ schemaDriftThreshold: schemaInfo.driftThreshold,
234
+ schemaMinAbsoluteDelta: schemaInfo.minAbsoluteDelta,
235
+ schemaMinRelativeDelta: schemaInfo.minRelativeDelta,
230
236
  templateConfig,
231
237
  assignmentConfig,
232
238
  });
@@ -252,7 +258,17 @@ export async function setupBaselineAnalyzerJob({
252
258
  logger.debug("Anomaly baseline analyzer job scheduled.");
253
259
  }
254
260
 
255
- function lookupSchemaDirection({
261
+ interface SchemaInfo {
262
+ direction?: AnomalyDirection;
263
+ sensitivity?: number;
264
+ confirmationWindow?: number;
265
+ driftEnabled?: boolean;
266
+ driftThreshold?: number;
267
+ minAbsoluteDelta?: number;
268
+ minRelativeDelta?: number;
269
+ }
270
+
271
+ function lookupSchemaInfo({
256
272
  collectorRegistry,
257
273
  collectorId,
258
274
  fieldName,
@@ -260,14 +276,22 @@ function lookupSchemaDirection({
260
276
  collectorRegistry: CollectorRegistry;
261
277
  collectorId: string;
262
278
  fieldName: string;
263
- }): AnomalyDirection | undefined {
279
+ }): SchemaInfo {
264
280
  const collector = collectorRegistry.getCollector(collectorId);
265
- if (!collector) return undefined;
281
+ if (!collector) return {};
266
282
  const collectorSchema = collector.collector.result.schema;
267
- if (!("shape" in collectorSchema)) return undefined;
283
+ if (!("shape" in collectorSchema)) return {};
268
284
  const shape = collectorSchema.shape as Record<string, z.ZodTypeAny>;
269
285
  const fieldSchema = shape[fieldName];
270
- if (!fieldSchema) return undefined;
286
+ if (!fieldSchema) return {};
271
287
  const meta = getHealthResultMeta(fieldSchema);
272
- return meta?.["x-anomaly-direction"];
288
+ return {
289
+ direction: meta?.["x-anomaly-direction"],
290
+ sensitivity: meta?.["x-anomaly-sensitivity"],
291
+ confirmationWindow: meta?.["x-anomaly-confirmation-window"],
292
+ driftEnabled: meta?.["x-anomaly-drift-enabled"],
293
+ driftThreshold: meta?.["x-anomaly-drift-threshold"],
294
+ minAbsoluteDelta: meta?.["x-anomaly-min-absolute-delta"],
295
+ minRelativeDelta: meta?.["x-anomaly-min-relative-delta"],
296
+ };
273
297
  }
package/src/plugin.ts CHANGED
@@ -16,6 +16,11 @@ import {
16
16
  } from "@checkstack/anomaly-common";
17
17
  import { specToRegistration } from "@checkstack/notification-common";
18
18
  import { HealthCheckApi } from "@checkstack/healthcheck-common";
19
+ import { entityKindExtensionPoint } from "@checkstack/gitops-backend";
20
+ import {
21
+ registerAnomalyGitOpsKinds,
22
+ registerAnomalyGitOpsDocumentation,
23
+ } from "./anomaly-gitops-kinds";
19
24
 
20
25
  import { definePluginMetadata } from "@checkstack/common";
21
26
 
@@ -37,6 +42,20 @@ export const plugin = createBackendPlugin({
37
42
 
38
43
  let routerCache: AnomalyRouterCache | undefined;
39
44
 
45
+ // ─── GitOps Entity Kind Registration ─────────────────────────────
46
+ // Mutable ref populated during init(); the reconciler closure pulls
47
+ // the service via the lazy accessor at sync time.
48
+ let gitopsService: AnomalyService | undefined;
49
+ const kindRegistry = env.getExtensionPoint(entityKindExtensionPoint);
50
+ registerAnomalyGitOpsKinds({
51
+ kindRegistry,
52
+ getService: () => {
53
+ if (!gitopsService)
54
+ throw new Error("AnomalyService not initialized");
55
+ return gitopsService;
56
+ },
57
+ });
58
+
40
59
  env.registerInit({
41
60
  schema,
42
61
  deps: {
@@ -71,6 +90,7 @@ export const plugin = createBackendPlugin({
71
90
  });
72
91
 
73
92
  const service = new AnomalyService(typedDb);
93
+ gitopsService = service;
74
94
  routerCache = createAnomalyRouterCache({ cacheManager, logger });
75
95
  const router = createRouter(service, logger, routerCache);
76
96
  rpc.registerRouter(router, anomalyContract);
@@ -97,6 +117,14 @@ export const plugin = createBackendPlugin({
97
117
  ),
98
118
  ]);
99
119
 
120
+ // GitOps spec-schema docs need the collector registry to be
121
+ // populated so we can enumerate per-collector result fields and
122
+ // register conditional variants under `anomaly.fieldOverrides`.
123
+ registerAnomalyGitOpsDocumentation({
124
+ kindRegistry,
125
+ collectorRegistry,
126
+ });
127
+
100
128
  onHook(healthCheckHooks.checkCompleted, async (payload) => {
101
129
  await processCheckCompleted({
102
130
  ...payload,
package/src/router.ts CHANGED
@@ -62,7 +62,7 @@ export function createRouter(
62
62
  getAnomalyAssignmentConfig: os.getAnomalyAssignmentConfig.handler(
63
63
  async ({ input }) => {
64
64
  const result = await service.getAnomalyAssignmentConfig(input.systemId, input.configurationId);
65
- // eslint-disable-next-line unicorn/no-null
65
+
66
66
  return (result as VersionedRecord<Partial<AnomalySettings>>) ?? null;
67
67
  }
68
68
  ),
package/src/service.ts CHANGED
@@ -44,9 +44,9 @@ export class AnomalyService {
44
44
  return results.map((r) => ({
45
45
  ...r,
46
46
  startedAt: r.startedAt.toISOString(),
47
- // eslint-disable-next-line unicorn/no-null
47
+
48
48
  confirmedAt: r.confirmedAt?.toISOString() ?? null,
49
- // eslint-disable-next-line unicorn/no-null
49
+
50
50
  recoveredAt: r.recoveredAt?.toISOString() ?? null,
51
51
  }));
52
52
  }
@@ -83,12 +83,8 @@ export class AnomalyService {
83
83
  // Return default configuration wrapper
84
84
  return anomalySettingsConfig.create({
85
85
  enabled: true,
86
- sensitivity: 1,
87
- confirmationWindow: 3,
88
86
  baselineWindow: "7d",
89
87
  notify: true,
90
- driftEnabled: true,
91
- driftThreshold: 2,
92
88
  });
93
89
  }
94
90
 
package/tsconfig.json CHANGED
@@ -28,6 +28,12 @@
28
28
  {
29
29
  "path": "../drizzle-helper"
30
30
  },
31
+ {
32
+ "path": "../gitops-backend"
33
+ },
34
+ {
35
+ "path": "../gitops-common"
36
+ },
31
37
  {
32
38
  "path": "../healthcheck-backend"
33
39
  },