@hogsend/cli 0.1.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin.js +575 -104
- package/dist/bin.js.map +1 -1
- package/package.json +4 -1
- package/skills/hogsend-authoring-buckets/SKILL.md +69 -0
- package/skills/hogsend-authoring-buckets/references/bucket-id-aliases.md +117 -0
- package/skills/hogsend-authoring-buckets/references/bucket-meta.md +142 -0
- package/skills/hogsend-authoring-buckets/references/buckets-vs-journeys.md +96 -0
- package/skills/hogsend-authoring-buckets/references/register-a-bucket.md +129 -0
- package/skills/hogsend-authoring-emails/SKILL.md +68 -0
- package/skills/hogsend-authoring-emails/references/email-components.md +112 -0
- package/skills/hogsend-authoring-emails/references/preview-and-render.md +116 -0
- package/skills/hogsend-authoring-emails/references/template-four-file-contract.md +134 -0
- package/skills/hogsend-authoring-emails/references/tracking-and-unsubscribe.md +127 -0
- package/skills/hogsend-authoring-journeys/SKILL.md +88 -0
- package/skills/hogsend-authoring-journeys/references/branch-on-engagement.md +117 -0
- package/skills/hogsend-authoring-journeys/references/journey-context.md +133 -0
- package/skills/hogsend-authoring-journeys/references/journey-meta.md +145 -0
- package/skills/hogsend-authoring-journeys/references/register-a-journey.md +99 -0
- package/skills/hogsend-authoring-journeys/references/sending-email-from-a-journey.md +82 -0
- package/skills/hogsend-cli/SKILL.md +1 -0
- package/skills/hogsend-conditions/SKILL.md +70 -0
- package/skills/hogsend-conditions/references/condition-types.md +251 -0
- package/skills/hogsend-conditions/references/durations.md +90 -0
- package/skills/hogsend-conditions/references/examples.md +188 -0
- package/skills/hogsend-database/SKILL.md +70 -0
- package/skills/hogsend-database/references/client-track-schema.md +97 -0
- package/skills/hogsend-database/references/migrations.md +132 -0
- package/skills/hogsend-database/references/schema-drift.md +123 -0
- package/skills/hogsend-deploy/SKILL.md +62 -0
- package/skills/hogsend-deploy/references/env-and-secrets.md +118 -0
- package/skills/hogsend-deploy/references/railway-two-services.md +122 -0
- package/skills/hogsend-deploy/references/upgrade-engine.md +92 -0
- package/skills/hogsend-webhooks-and-workflows/SKILL.md +68 -0
- package/skills/hogsend-webhooks-and-workflows/references/backfill-pattern.md +148 -0
- package/skills/hogsend-webhooks-and-workflows/references/custom-workflow.md +156 -0
- package/skills/hogsend-webhooks-and-workflows/references/webhook-source.md +172 -0
- package/src/commands/doctor.ts +22 -0
- package/src/commands/index.ts +4 -0
- package/src/commands/skills.ts +36 -96
- package/src/commands/studio.ts +261 -0
- package/src/commands/upgrade.ts +245 -0
- package/src/lib/skills.ts +186 -0
- package/studio/assets/index-BVA9GZqq.css +1 -0
- package/studio/assets/index-kPwzOOyG.js +230 -0
- package/studio/index.html +13 -0
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: hogsend-webhooks-and-workflows
|
|
3
|
+
description: Use when adding an inbound webhook source in src/webhook-sources/ (defineWebhookSource — auth header + envKey, optional Zod schema, transform(payload, ctx) -> IngestEvent | null, served at POST /v1/webhooks/:id) or a custom Hatchet task in src/workflows/ passed as extraWorkflows (NOT workflows) to createWorker, including the idempotent batched expand→migrate→contract backfill pattern.
|
|
4
|
+
license: MIT
|
|
5
|
+
metadata:
|
|
6
|
+
author: withSeismic
|
|
7
|
+
version: "1.0.0"
|
|
8
|
+
---
|
|
9
|
+
|
|
10
|
+
# Hogsend webhooks & workflows
|
|
11
|
+
|
|
12
|
+
This skill covers the two extension points a scaffolded Hogsend app uses to take
|
|
13
|
+
in external events and run background jobs:
|
|
14
|
+
|
|
15
|
+
1. **Webhook sources** — turn an inbound HTTP payload into an `IngestEvent` that
|
|
16
|
+
flows through the engine's ingestion pipeline (and can trigger journeys).
|
|
17
|
+
2. **Custom Hatchet tasks** — durable background work (one-off jobs, backfills,
|
|
18
|
+
cron-style maintenance) registered alongside the engine's built-in workflows.
|
|
19
|
+
|
|
20
|
+
You are editing a **content-only consumer**: you import everything from
|
|
21
|
+
`@hogsend/engine` (and `@hogsend/db` for tasks). Never edit engine internals.
|
|
22
|
+
Relative imports use the ESM `.js` extension.
|
|
23
|
+
|
|
24
|
+
## Capability map / key concepts
|
|
25
|
+
|
|
26
|
+
- **`defineWebhookSource({ meta, auth, schema?, transform })`** (from
|
|
27
|
+
`@hogsend/engine`) — declares one source served at `POST /v1/webhooks/:id`.
|
|
28
|
+
`auth` matches a request header against an env secret; `schema` is an optional
|
|
29
|
+
Zod validator; `transform(payload, ctx)` returns an `IngestEvent | null`
|
|
30
|
+
(`null` = accept-and-skip). Register sources in `src/webhook-sources/index.ts`
|
|
31
|
+
and pass them to `createApp(client, { webhookSources })` in `src/index.ts`.
|
|
32
|
+
- **`IngestEvent`** — the shape `transform` must return:
|
|
33
|
+
`{ event, userId, userEmail, properties, idempotencyKey? }`. The route feeds it
|
|
34
|
+
straight into `ingestEvent()`, so a webhook can enroll users into journeys.
|
|
35
|
+
- **Custom Hatchet tasks** — define with `hatchet.task({ name, fn })` (or
|
|
36
|
+
`hatchet.durableTask` for event-driven/long-running work), export from
|
|
37
|
+
`src/workflows/index.ts` in the `extraWorkflows` array, and pass it as
|
|
38
|
+
`createWorker({ ..., extraWorkflows })` — the option is **`extraWorkflows`,
|
|
39
|
+
NOT `workflows`**. Never list the engine's built-ins (send-email,
|
|
40
|
+
import-contacts, check-alerts, bucket tasks) — those register automatically.
|
|
41
|
+
- **JSON-serializable IO** — task input AND return value must serialize to JSON.
|
|
42
|
+
Use specific keys or `JsonValue`-compatible types; do **not** use a
|
|
43
|
+
`[key: string]: unknown` index signature.
|
|
44
|
+
- **Backfill pattern** — `runBatchedBackfill()` (from `@hogsend/engine`) drives a
|
|
45
|
+
long data migration in small, idempotent, lock-friendly batches from inside a
|
|
46
|
+
task — the supported home for bulk data changes (never inside a schema
|
|
47
|
+
migration). Follow expand → migrate → contract across releases.
|
|
48
|
+
|
|
49
|
+
## Task playbooks — load the matching reference
|
|
50
|
+
|
|
51
|
+
- **Adding / editing an inbound webhook source** → load
|
|
52
|
+
`references/webhook-source.md` (defineWebhookSource fields, the `transform` →
|
|
53
|
+
`ingestEvent` contract, auth matching, registration + `createApp` wiring).
|
|
54
|
+
- **Writing a custom Hatchet task (one-off job, cron, event-driven)** → load
|
|
55
|
+
`references/custom-workflow.md` (`hatchet.task`/`durableTask`,
|
|
56
|
+
JSON-serializable IO, export from `index.ts`, `createWorker({ extraWorkflows })`).
|
|
57
|
+
- **Backfilling a new column on existing rows** → load
|
|
58
|
+
`references/backfill-pattern.md` (the idempotent batched
|
|
59
|
+
expand→migrate→contract job from the template example).
|
|
60
|
+
|
|
61
|
+
## Cross-skill pointers
|
|
62
|
+
|
|
63
|
+
- A webhook's `transform` only needs to emit the right `event`/`properties`;
|
|
64
|
+
whether a journey then enrolls or exits is decided by trigger/exit conditions —
|
|
65
|
+
see the **hogsend-conditions** skill for `where`/`exitOn`/criteria and duration
|
|
66
|
+
helpers.
|
|
67
|
+
- To verify a webhook or task against a running instance (events landing,
|
|
68
|
+
contacts upserted, journeys firing), see the **hogsend-cli** skill.
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
# Reference: the idempotent batched backfill
|
|
2
|
+
|
|
3
|
+
When a release adds a column that needs populating on existing rows, **do not**
|
|
4
|
+
put the data change inside a schema migration — that holds locks and runs
|
|
5
|
+
unbounded against a live database. Instead, drive it from a Hatchet task using
|
|
6
|
+
`runBatchedBackfill` (from `@hogsend/engine`), which runs the migration in small,
|
|
7
|
+
idempotent, lock-friendly batches that are **resumable**: if the process dies,
|
|
8
|
+
re-running continues from where it left off because each batch only selects rows
|
|
9
|
+
that still need work.
|
|
10
|
+
|
|
11
|
+
The scaffold ships a ready-to-customize template at
|
|
12
|
+
`src/workflows/backfill-example.ts`.
|
|
13
|
+
|
|
14
|
+
## Expand → migrate → contract
|
|
15
|
+
|
|
16
|
+
Sequence the change across releases so old and new code can run side by side:
|
|
17
|
+
|
|
18
|
+
1. **Release N (expand)** — the migration adds the column (nullable / defaulted);
|
|
19
|
+
code writes BOTH the old and new shape. Deploy.
|
|
20
|
+
2. **Run the backfill task once** (Hatchet dashboard, or push its event) to
|
|
21
|
+
populate existing rows. It's batched, idempotent and resumable.
|
|
22
|
+
3. **Release N+1** — code reads the new column.
|
|
23
|
+
4. **Release N+2 (contract)** — once the backfill is confirmed complete, a
|
|
24
|
+
migration drops the old column / adds `NOT NULL`.
|
|
25
|
+
|
|
26
|
+
## `runBatchedBackfill` — the driver
|
|
27
|
+
|
|
28
|
+
```ts
|
|
29
|
+
interface BatchedBackfillOptions {
|
|
30
|
+
db: Database;
|
|
31
|
+
logger: { info: (m: string) => unknown; warn: (m: string) => unknown };
|
|
32
|
+
label: string; // human label for logs, e.g. "contacts.normalized_email"
|
|
33
|
+
runBatch: (db: Database, batchSize: number) => Promise<number>; // rows affected; 0 = done
|
|
34
|
+
batchSize?: number; // default 500
|
|
35
|
+
pauseMs?: number; // default 0 — pause between batches to relieve a live DB
|
|
36
|
+
maxBatches?: number; // default 100_000 — safety cap, logs + stops (not an error)
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
interface BatchedBackfillResult {
|
|
40
|
+
batches: number;
|
|
41
|
+
rows: number;
|
|
42
|
+
exhausted: boolean; // true only when a batch returned 0 (ran to completion)
|
|
43
|
+
}
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
The two rules that make it safe:
|
|
47
|
+
|
|
48
|
+
- **`runBatch` MUST be idempotent and self-bounding.** It should touch only rows
|
|
49
|
+
that still need the change (e.g. `WHERE new_col IS NULL ... LIMIT n`) and return
|
|
50
|
+
the number of rows affected. Return `0` when nothing is left — that's the signal
|
|
51
|
+
to stop (`exhausted: true`).
|
|
52
|
+
- **Lock-friendly batches.** Select the batch with `FOR UPDATE SKIP LOCKED` so
|
|
53
|
+
concurrent runs/workers don't fight over the same rows, and keep `batchSize`
|
|
54
|
+
small so each statement holds locks only briefly.
|
|
55
|
+
|
|
56
|
+
## The template task
|
|
57
|
+
|
|
58
|
+
`src/workflows/backfill-example.ts` — assumes a release just added
|
|
59
|
+
`contacts.normalized_email`. Change the table/columns to match your migration:
|
|
60
|
+
|
|
61
|
+
```ts
|
|
62
|
+
import { createDatabase } from "@hogsend/db";
|
|
63
|
+
import { createLogger, hatchet, runBatchedBackfill } from "@hogsend/engine";
|
|
64
|
+
import { sql } from "drizzle-orm";
|
|
65
|
+
|
|
66
|
+
export const backfillExampleTask = hatchet.task({
|
|
67
|
+
name: "backfill-example",
|
|
68
|
+
retries: 2,
|
|
69
|
+
executionTimeout: "30m",
|
|
70
|
+
fn: async () => {
|
|
71
|
+
const { db, client } = createDatabase({ url: process.env.DATABASE_URL ?? "" });
|
|
72
|
+
const logger = createLogger(process.env.LOG_LEVEL ?? "info");
|
|
73
|
+
|
|
74
|
+
try {
|
|
75
|
+
const result = await runBatchedBackfill({
|
|
76
|
+
db,
|
|
77
|
+
logger,
|
|
78
|
+
label: "contacts.normalized_email",
|
|
79
|
+
batchSize: 500,
|
|
80
|
+
pauseMs: 50,
|
|
81
|
+
runBatch: async (database, limit) => {
|
|
82
|
+
// Bounded, idempotent batch: only rows that still need it, locked with
|
|
83
|
+
// SKIP LOCKED so concurrent runs/workers don't fight over the same rows.
|
|
84
|
+
const updated = (await database.execute(sql`
|
|
85
|
+
WITH batch AS (
|
|
86
|
+
SELECT id FROM contacts
|
|
87
|
+
WHERE normalized_email IS NULL
|
|
88
|
+
LIMIT ${limit}
|
|
89
|
+
FOR UPDATE SKIP LOCKED
|
|
90
|
+
)
|
|
91
|
+
UPDATE contacts c
|
|
92
|
+
SET normalized_email = lower(trim(c.email))
|
|
93
|
+
FROM batch
|
|
94
|
+
WHERE c.id = batch.id
|
|
95
|
+
RETURNING c.id
|
|
96
|
+
`)) as unknown as unknown[];
|
|
97
|
+
return updated.length;
|
|
98
|
+
},
|
|
99
|
+
});
|
|
100
|
+
|
|
101
|
+
// Return a plain JSON object so Hatchet can serialize the task output.
|
|
102
|
+
return {
|
|
103
|
+
batches: result.batches,
|
|
104
|
+
rows: result.rows,
|
|
105
|
+
exhausted: result.exhausted,
|
|
106
|
+
};
|
|
107
|
+
} finally {
|
|
108
|
+
await client.end({ timeout: 5 });
|
|
109
|
+
}
|
|
110
|
+
},
|
|
111
|
+
});
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
Anatomy of the batch SQL above:
|
|
115
|
+
|
|
116
|
+
- The `batch` CTE selects up to `limit` rows that **still need work**
|
|
117
|
+
(`WHERE normalized_email IS NULL`) and locks them with `FOR UPDATE SKIP LOCKED`.
|
|
118
|
+
- The `UPDATE ... FROM batch` writes only those locked rows and `RETURNING c.id`
|
|
119
|
+
gives a row count — `updated.length` is what `runBatch` returns.
|
|
120
|
+
- Because the predicate excludes already-migrated rows, a re-run after a crash
|
|
121
|
+
resumes cleanly, and two workers never collide.
|
|
122
|
+
|
|
123
|
+
## Enabling it
|
|
124
|
+
|
|
125
|
+
The example is **already wired** via `extraWorkflows` in `src/workflows/index.ts`:
|
|
126
|
+
|
|
127
|
+
```ts
|
|
128
|
+
import { backfillExampleTask } from "./backfill-example.js";
|
|
129
|
+
|
|
130
|
+
export const extraWorkflows = [backfillExampleTask];
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
which `src/worker.ts` passes as `createWorker({ container, journeys, buckets,
|
|
134
|
+
extraWorkflows })`. (Remove `backfillExampleTask` and the file if you don't need
|
|
135
|
+
it.) For the general task-registration mechanics see
|
|
136
|
+
`references/custom-workflow.md`. Trigger the run once from the Hatchet dashboard
|
|
137
|
+
(or by pushing its event); re-running is safe and resumes where it left off.
|
|
138
|
+
|
|
139
|
+
## Adapting it — checklist
|
|
140
|
+
|
|
141
|
+
1. Copy `backfill-example.ts` (or edit it) to target your new table/column.
|
|
142
|
+
2. Make `runBatch`'s predicate exclude already-done rows (`WHERE new_col IS NULL`)
|
|
143
|
+
and lock with `FOR UPDATE SKIP LOCKED`.
|
|
144
|
+
3. Keep `batchSize` modest; set `pauseMs` to relieve a live database.
|
|
145
|
+
4. Return the plain `{ batches, rows, exhausted }` summary (JSON-serializable).
|
|
146
|
+
5. Ensure the task is listed in `extraWorkflows`, restart the worker, run it once.
|
|
147
|
+
6. Only after `exhausted: true` is confirmed, ship the contract migration
|
|
148
|
+
(`NOT NULL` / drop old column).
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
# Reference: custom Hatchet tasks (`extraWorkflows`)
|
|
2
|
+
|
|
3
|
+
Custom tasks are durable background jobs you own — one-off maintenance, backfills,
|
|
4
|
+
cron-style work, or event-driven side effects. They run in the **worker** process
|
|
5
|
+
alongside the engine's built-in workflows. You define them in `src/workflows/`,
|
|
6
|
+
export them from `src/workflows/index.ts`, and the scaffold passes them to
|
|
7
|
+
`createWorker({ container, journeys, extraWorkflows })`.
|
|
8
|
+
|
|
9
|
+
The option is **`extraWorkflows` — NOT `workflows`.** The engine registers its own
|
|
10
|
+
built-ins (`send-email`, `import-contacts`, `check-alerts`, and the bucket tasks)
|
|
11
|
+
automatically; `extraWorkflows` is *additive*. Never list a built-in there.
|
|
12
|
+
|
|
13
|
+
## Defining a task
|
|
14
|
+
|
|
15
|
+
Import the shared `hatchet` client from `@hogsend/engine`. Two flavours:
|
|
16
|
+
|
|
17
|
+
- `hatchet.task({ name, fn })` — a plain task you trigger explicitly (one-off
|
|
18
|
+
jobs, backfills, anything kicked off from the dashboard or via `hatchet.events`).
|
|
19
|
+
- `hatchet.durableTask({ name, onEvents, fn })` — long-running / event-driven work
|
|
20
|
+
(this is what journeys use under the hood). Declare `onEvents: [eventName]` to
|
|
21
|
+
have Hatchet route ingested events to the task automatically.
|
|
22
|
+
|
|
23
|
+
### JSON-serializable IO (hard requirement)
|
|
24
|
+
|
|
25
|
+
A task's **input** and **return value** must serialize to JSON.
|
|
26
|
+
|
|
27
|
+
- Use specific, named keys (`{ jobId: string; format: string }`) or
|
|
28
|
+
`JsonValue`-compatible types.
|
|
29
|
+
- Do **NOT** use a `[key: string]: unknown` index signature on the input type.
|
|
30
|
+
- Return a plain object (or a small JSON-safe value) so Hatchet can store the
|
|
31
|
+
task output — return `void`/`undefined` if there's nothing to report.
|
|
32
|
+
|
|
33
|
+
```ts
|
|
34
|
+
import { hatchet } from "@hogsend/engine";
|
|
35
|
+
|
|
36
|
+
// input + return are both plain, named-key JSON objects
|
|
37
|
+
export const reindexSearchTask = hatchet.task({
|
|
38
|
+
name: "reindex-search",
|
|
39
|
+
retries: 2,
|
|
40
|
+
executionTimeout: "30m",
|
|
41
|
+
fn: async (input: { since: string; dryRun: boolean }) => {
|
|
42
|
+
// ... do the work ...
|
|
43
|
+
return { reindexed: 0, skipped: 0, dryRun: input.dryRun };
|
|
44
|
+
},
|
|
45
|
+
});
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
The engine's own `import-contacts` task is a faithful template for a parameterized
|
|
49
|
+
job — a named-key input, batched processing, and a JSON return:
|
|
50
|
+
|
|
51
|
+
```ts
|
|
52
|
+
export const importContactsTask = hatchet.task({
|
|
53
|
+
name: "import-contacts",
|
|
54
|
+
retries: 0,
|
|
55
|
+
executionTimeout: "600s",
|
|
56
|
+
fn: async (input: { jobId: string; data: string; format: string }) => {
|
|
57
|
+
// ...batched upserts...
|
|
58
|
+
return { status: "completed", processed, failed };
|
|
59
|
+
},
|
|
60
|
+
});
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
### Event-driven variant
|
|
64
|
+
|
|
65
|
+
If the task should run whenever a particular event is ingested, use a durable task
|
|
66
|
+
with `onEvents`. The input arrives as the ingested event payload
|
|
67
|
+
(`userId` / `userEmail` / scalar `properties`):
|
|
68
|
+
|
|
69
|
+
```ts
|
|
70
|
+
import { hatchet } from "@hogsend/engine";
|
|
71
|
+
|
|
72
|
+
export const onSignupAuditTask = hatchet.durableTask({
|
|
73
|
+
name: "on-signup-audit",
|
|
74
|
+
onEvents: ["user.signed_up"],
|
|
75
|
+
executionTimeout: "10m",
|
|
76
|
+
retries: 1,
|
|
77
|
+
fn: async (input: {
|
|
78
|
+
userId: string;
|
|
79
|
+
userEmail: string;
|
|
80
|
+
properties: Record<string, string | number | boolean | null>;
|
|
81
|
+
}) => {
|
|
82
|
+
// side effect; return a JSON-safe summary (or nothing)
|
|
83
|
+
return { audited: input.userId };
|
|
84
|
+
},
|
|
85
|
+
});
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
> For normal lifecycle messaging, prefer a journey (`defineJourney`) over a raw
|
|
89
|
+
> durable task — journeys give you enrollment guards, state tracking, durable
|
|
90
|
+
> sleeps and exit conditions for free. Reach for a custom durable task only when
|
|
91
|
+
> you need orchestration the journey system doesn't model.
|
|
92
|
+
|
|
93
|
+
## Accessing the database inside a task
|
|
94
|
+
|
|
95
|
+
Tasks run in the worker and are constructed at module load, so they don't receive
|
|
96
|
+
the request `container`. Open a connection inside `fn` with `createDatabase` from
|
|
97
|
+
`@hogsend/db` and **always close it** in a `finally`:
|
|
98
|
+
|
|
99
|
+
```ts
|
|
100
|
+
import { createDatabase } from "@hogsend/db";
|
|
101
|
+
import { createLogger, hatchet } from "@hogsend/engine";
|
|
102
|
+
|
|
103
|
+
export const nightlyCleanupTask = hatchet.task({
|
|
104
|
+
name: "nightly-cleanup",
|
|
105
|
+
fn: async () => {
|
|
106
|
+
const { db, client } = createDatabase({ url: process.env.DATABASE_URL ?? "" });
|
|
107
|
+
const logger = createLogger(process.env.LOG_LEVEL ?? "info");
|
|
108
|
+
try {
|
|
109
|
+
// ...work with db...
|
|
110
|
+
return { ok: true };
|
|
111
|
+
} finally {
|
|
112
|
+
await client.end({ timeout: 5 });
|
|
113
|
+
}
|
|
114
|
+
},
|
|
115
|
+
});
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
## Wiring it up (two edits)
|
|
119
|
+
|
|
120
|
+
### 1. Export from `src/workflows/index.ts`
|
|
121
|
+
|
|
122
|
+
List only YOUR tasks here — the engine adds its built-ins itself:
|
|
123
|
+
|
|
124
|
+
```ts
|
|
125
|
+
import { backfillExampleTask } from "./backfill-example.js";
|
|
126
|
+
import { nightlyCleanupTask } from "./nightly-cleanup.js";
|
|
127
|
+
|
|
128
|
+
export const extraWorkflows = [backfillExampleTask, nightlyCleanupTask];
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
### 2. Confirm `src/worker.ts` passes it
|
|
132
|
+
|
|
133
|
+
The scaffold already does this — the key detail is the option name:
|
|
134
|
+
|
|
135
|
+
```ts
|
|
136
|
+
const worker = createWorker({
|
|
137
|
+
container: client,
|
|
138
|
+
journeys,
|
|
139
|
+
buckets,
|
|
140
|
+
extraWorkflows, // <-- NOT `workflows`
|
|
141
|
+
});
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
`createWorker` builds the worker as `[...engine built-ins, ...journeyTasks,
|
|
145
|
+
...bucketTasks, ...extraWorkflows]`. After editing, restart the worker
|
|
146
|
+
(`hatchet worker dev` or `pnpm worker:dev`) so the new task registers.
|
|
147
|
+
|
|
148
|
+
## Authoring a new task — checklist
|
|
149
|
+
|
|
150
|
+
1. Create `src/workflows/<name>.ts` exporting `hatchet.task({...})` (or
|
|
151
|
+
`hatchet.durableTask` for event-driven/long-running work).
|
|
152
|
+
2. Type the input with named keys (no `[key: string]: unknown`); return JSON-safe
|
|
153
|
+
data.
|
|
154
|
+
3. Open `createDatabase(...)` inside `fn` and close it in `finally`.
|
|
155
|
+
4. Add the export to the `extraWorkflows` array in `src/workflows/index.ts`.
|
|
156
|
+
5. Restart the worker; trigger the task (Hatchet dashboard or `hatchet.events`).
|
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
# Reference: webhook sources (`defineWebhookSource`)
|
|
2
|
+
|
|
3
|
+
A webhook source turns an inbound HTTP request into an `IngestEvent`. Each source
|
|
4
|
+
is served by the engine at **`POST /v1/webhooks/{sourceId}`** and its output is
|
|
5
|
+
fed straight into `ingestEvent()` — so a webhook can store an event, push it to
|
|
6
|
+
Hatchet (routing it to matching journey tasks), evaluate journey exit conditions,
|
|
7
|
+
and upsert the contact.
|
|
8
|
+
|
|
9
|
+
You write the source files; the engine owns the route. Edit only
|
|
10
|
+
`src/webhook-sources/`.
|
|
11
|
+
|
|
12
|
+
## The shape you implement
|
|
13
|
+
|
|
14
|
+
`defineWebhookSource` is imported from `@hogsend/engine`. The definition object:
|
|
15
|
+
|
|
16
|
+
| Field | Type | Notes |
|
|
17
|
+
|-------|------|-------|
|
|
18
|
+
| `meta.id` | `string` | The `:sourceId` segment in the URL. Keep it URL-safe. |
|
|
19
|
+
| `meta.name` | `string` | Human label. |
|
|
20
|
+
| `meta.description?` | `string` | Optional. |
|
|
21
|
+
| `auth.header` | `string` | Request header carrying the shared secret. |
|
|
22
|
+
| `auth.envKey` | `string` | Env var holding the expected secret value. |
|
|
23
|
+
| `auth.type` | `"match"` | Only mode today: header value must equal the env value. |
|
|
24
|
+
| `schema?` | `z.ZodSchema<T>` | Optional Zod validator; on success `payload` is typed `T`. |
|
|
25
|
+
| `transform(payload, ctx)` | `=> Promise<IngestEvent \| null>` | Map payload → event. Return `null` to accept-and-skip. |
|
|
26
|
+
|
|
27
|
+
### Auth behaviour (important)
|
|
28
|
+
|
|
29
|
+
The route enforces auth **only when the env secret is set**. If
|
|
30
|
+
`process.env[auth.envKey]` is empty/undefined the source is treated as **open**
|
|
31
|
+
(no auth). When the secret is present, the request must send it either in
|
|
32
|
+
`auth.header` or as `Authorization: Bearer <secret>`; otherwise the route returns
|
|
33
|
+
`401`. Always set the env secret in any non-local environment.
|
|
34
|
+
|
|
35
|
+
### Validation
|
|
36
|
+
|
|
37
|
+
If you provide `schema`, the route runs `schema.safeParse(payload)` before
|
|
38
|
+
calling `transform`; a parse failure returns `400` and `transform` never runs.
|
|
39
|
+
Inside `transform`, `payload` is the parsed, typed value.
|
|
40
|
+
|
|
41
|
+
## The `transform` → `IngestEvent` contract
|
|
42
|
+
|
|
43
|
+
`transform(payload, ctx)` returns an `IngestEvent` (or `null`):
|
|
44
|
+
|
|
45
|
+
```ts
|
|
46
|
+
interface IngestEvent {
|
|
47
|
+
event: string; // event name (this is what journeys trigger on)
|
|
48
|
+
userId: string; // external/distinct id of the person
|
|
49
|
+
userEmail: string; // "" if unknown — emptystring, not undefined
|
|
50
|
+
properties: Record<string, unknown>; // event + person props; merged into the event
|
|
51
|
+
idempotencyKey?: string; // optional dedupe key (see below)
|
|
52
|
+
}
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
`ctx` is `{ db, logger }` — a Drizzle `Database` and the engine logger — for
|
|
56
|
+
lookups/diagnostics inside the transform. It does **not** carry `hatchet` or the
|
|
57
|
+
registry; those are applied by the route when it calls `ingestEvent`.
|
|
58
|
+
|
|
59
|
+
Notes that match the engine's behaviour:
|
|
60
|
+
|
|
61
|
+
- `event` is the routing key. Hatchet routes the pushed event to every journey
|
|
62
|
+
whose trigger declares `onEvents: [thatEvent]`. The decision to *enroll* (or
|
|
63
|
+
*exit*) is then made by trigger/exit conditions — see the **hogsend-conditions**
|
|
64
|
+
skill.
|
|
65
|
+
- `userEmail` should be `""` when unknown (the ingestion pipeline treats a falsy
|
|
66
|
+
email as "no email" for the contact upsert). Don't pass `undefined`.
|
|
67
|
+
- Only JSON-scalar properties (`string | number | boolean | null`) survive the
|
|
68
|
+
push to Hatchet; nested objects/arrays are dropped from the event payload that
|
|
69
|
+
reaches journey tasks (they're still stored on the `userEvents` row). Flatten
|
|
70
|
+
anything a journey needs to branch on into a scalar property.
|
|
71
|
+
- `idempotencyKey` (optional): when set, a duplicate delivery with the same key is
|
|
72
|
+
a no-op (`{ stored: false }`) — use the provider's event id when available.
|
|
73
|
+
- Return `null` to accept the delivery (HTTP `200 { ok: true, skipped: true }`)
|
|
74
|
+
without ingesting — e.g. event types you don't care about.
|
|
75
|
+
|
|
76
|
+
## Example: a source from the scaffold
|
|
77
|
+
|
|
78
|
+
`src/webhook-sources/posthog.ts` — validates a PostHog destination payload and
|
|
79
|
+
maps it to an `IngestEvent`:
|
|
80
|
+
|
|
81
|
+
```ts
|
|
82
|
+
import { defineWebhookSource } from "@hogsend/engine";
|
|
83
|
+
import { z } from "zod";
|
|
84
|
+
|
|
85
|
+
const posthogWebhookSchema = z.object({
|
|
86
|
+
event: z.object({
|
|
87
|
+
uuid: z.string().optional(),
|
|
88
|
+
event: z.string(),
|
|
89
|
+
distinct_id: z.string(),
|
|
90
|
+
properties: z.record(z.string(), z.unknown()).optional(),
|
|
91
|
+
}),
|
|
92
|
+
person: z
|
|
93
|
+
.object({
|
|
94
|
+
properties: z
|
|
95
|
+
.object({ email: z.string().optional() })
|
|
96
|
+
.catchall(z.unknown())
|
|
97
|
+
.optional(),
|
|
98
|
+
})
|
|
99
|
+
.optional(),
|
|
100
|
+
});
|
|
101
|
+
|
|
102
|
+
export const posthogSource = defineWebhookSource({
|
|
103
|
+
meta: {
|
|
104
|
+
id: "posthog",
|
|
105
|
+
name: "PostHog",
|
|
106
|
+
description: "Receives events from PostHog webhook destinations.",
|
|
107
|
+
},
|
|
108
|
+
auth: {
|
|
109
|
+
header: "x-posthog-webhook-secret",
|
|
110
|
+
envKey: "POSTHOG_WEBHOOK_SECRET",
|
|
111
|
+
type: "match",
|
|
112
|
+
},
|
|
113
|
+
schema: posthogWebhookSchema,
|
|
114
|
+
async transform(payload) {
|
|
115
|
+
const rawEmail = payload.person?.properties?.email;
|
|
116
|
+
const userEmail = typeof rawEmail === "string" ? rawEmail : "";
|
|
117
|
+
|
|
118
|
+
const properties: Record<string, unknown> = {
|
|
119
|
+
...payload.event.properties,
|
|
120
|
+
...payload.person?.properties,
|
|
121
|
+
};
|
|
122
|
+
if (payload.event.uuid) {
|
|
123
|
+
properties._posthogEventId = payload.event.uuid;
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
return {
|
|
127
|
+
event: payload.event.event,
|
|
128
|
+
userId: payload.event.distinct_id,
|
|
129
|
+
userEmail,
|
|
130
|
+
properties,
|
|
131
|
+
};
|
|
132
|
+
},
|
|
133
|
+
});
|
|
134
|
+
```
|
|
135
|
+
|
|
136
|
+
## Wiring it up (two edits)
|
|
137
|
+
|
|
138
|
+
### 1. Register in `src/webhook-sources/index.ts`
|
|
139
|
+
|
|
140
|
+
```ts
|
|
141
|
+
import type { DefinedWebhookSource } from "@hogsend/engine";
|
|
142
|
+
import { posthogSource } from "./posthog.js";
|
|
143
|
+
import { stripeSource } from "./stripe.js"; // your new source
|
|
144
|
+
|
|
145
|
+
export const webhookSources: DefinedWebhookSource[] = [
|
|
146
|
+
posthogSource,
|
|
147
|
+
stripeSource,
|
|
148
|
+
];
|
|
149
|
+
```
|
|
150
|
+
|
|
151
|
+
### 2. Pass to `createApp` in `src/index.ts`
|
|
152
|
+
|
|
153
|
+
The scaffold already threads this — confirm it's present:
|
|
154
|
+
|
|
155
|
+
```ts
|
|
156
|
+
import { webhookSources } from "./webhook-sources/index.js";
|
|
157
|
+
|
|
158
|
+
const app = createApp(client, { webhookSources });
|
|
159
|
+
```
|
|
160
|
+
|
|
161
|
+
That's it. Your source is now live at `POST /v1/webhooks/stripe`.
|
|
162
|
+
|
|
163
|
+
## Authoring a new source — checklist
|
|
164
|
+
|
|
165
|
+
1. Create `src/webhook-sources/<id>.ts` exporting a `defineWebhookSource({...})`.
|
|
166
|
+
2. Pick a unique `meta.id` (becomes the URL segment).
|
|
167
|
+
3. Set `auth.envKey` and add that secret to your env for non-local deploys.
|
|
168
|
+
4. Add a Zod `schema` for the payload you expect (recommended).
|
|
169
|
+
5. In `transform`, produce `{ event, userId, userEmail, properties }` (or `null`).
|
|
170
|
+
Flatten anything a journey will branch on into a scalar property.
|
|
171
|
+
6. Add the export to the `webhookSources` array in `index.ts`.
|
|
172
|
+
7. Verify deliveries land using the **hogsend-cli** skill (events/contacts).
|
package/src/commands/doctor.ts
CHANGED
|
@@ -1,8 +1,27 @@
|
|
|
1
1
|
import { parseArgs } from "node:util";
|
|
2
2
|
import { isHttpError } from "../lib/http.js";
|
|
3
3
|
import { color } from "../lib/output.js";
|
|
4
|
+
import { skillsStaleness } from "../lib/skills.js";
|
|
4
5
|
import type { Command, CommandContext } from "./types.js";
|
|
5
6
|
|
|
7
|
+
/**
|
|
8
|
+
* Best-effort nudge: if the cwd is a Hogsend app whose vendored skills were
|
|
9
|
+
* installed by an OLDER CLI than the one running now, point the user at the
|
|
10
|
+
* refresh. Silent when there's no stamp (not an app dir / never tracked).
|
|
11
|
+
*/
|
|
12
|
+
function skillsNudge(ctx: CommandContext): void {
|
|
13
|
+
const verdict = skillsStaleness(process.cwd());
|
|
14
|
+
if (!verdict?.stale || ctx.json) return;
|
|
15
|
+
ctx.out.note(
|
|
16
|
+
[
|
|
17
|
+
`Vendored Claude skills are from v${verdict.installed}; this CLI is v${verdict.current}.`,
|
|
18
|
+
"",
|
|
19
|
+
`Refresh: ${color.cyan("hogsend upgrade")} ${color.dim("(deps + skills)")} or ${color.cyan("hogsend skills add --all --force")}.`,
|
|
20
|
+
].join("\n"),
|
|
21
|
+
"Skills out of date",
|
|
22
|
+
);
|
|
23
|
+
}
|
|
24
|
+
|
|
6
25
|
const usage = `hogsend doctor [--url <baseUrl>] [--admin-key <key>] [--json]
|
|
7
26
|
|
|
8
27
|
Probe a running Hogsend instance via GET /v1/health and report its health:
|
|
@@ -159,6 +178,7 @@ async function run(ctx: CommandContext): Promise<void> {
|
|
|
159
178
|
timestamp: health.timestamp,
|
|
160
179
|
components: health.components,
|
|
161
180
|
schema: health.schema,
|
|
181
|
+
skills: skillsStaleness(process.cwd()) ?? undefined,
|
|
162
182
|
});
|
|
163
183
|
if (!ok) process.exit(1);
|
|
164
184
|
return;
|
|
@@ -200,6 +220,8 @@ async function run(ctx: CommandContext): Promise<void> {
|
|
|
200
220
|
|
|
201
221
|
ctx.out.note(lines.join("\n"), "Doctor");
|
|
202
222
|
|
|
223
|
+
skillsNudge(ctx);
|
|
224
|
+
|
|
203
225
|
if (ok) {
|
|
204
226
|
ctx.out.outro(color.green("doctor: ok"));
|
|
205
227
|
return;
|
package/src/commands/index.ts
CHANGED
|
@@ -7,7 +7,9 @@ import { patchCommand } from "./patch.js";
|
|
|
7
7
|
import { setupCommand } from "./setup.js";
|
|
8
8
|
import { skillsCommand } from "./skills.js";
|
|
9
9
|
import { statsCommand } from "./stats.js";
|
|
10
|
+
import { studioCommand } from "./studio.js";
|
|
10
11
|
import type { Command } from "./types.js";
|
|
12
|
+
import { upgradeCommand } from "./upgrade.js";
|
|
11
13
|
|
|
12
14
|
/**
|
|
13
15
|
* The command registry. The router (src/bin.ts) matches the leading argv token
|
|
@@ -23,8 +25,10 @@ export const commands: Command[] = [
|
|
|
23
25
|
contactsCommand,
|
|
24
26
|
statsCommand,
|
|
25
27
|
eventsCommand,
|
|
28
|
+
studioCommand,
|
|
26
29
|
setupCommand,
|
|
27
30
|
skillsCommand,
|
|
31
|
+
upgradeCommand,
|
|
28
32
|
ejectCommand,
|
|
29
33
|
patchCommand,
|
|
30
34
|
];
|