kitfly 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +60 -0
- package/LICENSE +21 -0
- package/README.md +136 -0
- package/VERSION +1 -0
- package/package.json +63 -0
- package/schemas/README.md +32 -0
- package/schemas/site.schema.json +5 -0
- package/schemas/theme.schema.json +5 -0
- package/schemas/v0/site.schema.json +172 -0
- package/schemas/v0/theme.schema.json +210 -0
- package/scripts/build-all.ts +121 -0
- package/scripts/build.ts +601 -0
- package/scripts/bundle.ts +781 -0
- package/scripts/dev.ts +777 -0
- package/scripts/generate-checksums.sh +78 -0
- package/scripts/release/export-release-key.sh +28 -0
- package/scripts/release/release-guard-tag-version.sh +79 -0
- package/scripts/release/sign-release-assets.sh +123 -0
- package/scripts/release/upload-release-assets.sh +76 -0
- package/scripts/release/upload-release-provenance.sh +52 -0
- package/scripts/release/verify-public-key.sh +48 -0
- package/scripts/release/verify-signatures.sh +117 -0
- package/scripts/version-sync.ts +82 -0
- package/src/__tests__/build.test.ts +240 -0
- package/src/__tests__/bundle.test.ts +786 -0
- package/src/__tests__/cli.test.ts +706 -0
- package/src/__tests__/crucible.test.ts +1043 -0
- package/src/__tests__/engine.test.ts +157 -0
- package/src/__tests__/init.test.ts +450 -0
- package/src/__tests__/pipeline.test.ts +1087 -0
- package/src/__tests__/productbook.test.ts +1206 -0
- package/src/__tests__/runbook.test.ts +974 -0
- package/src/__tests__/server-registry.test.ts +1251 -0
- package/src/__tests__/servicebook.test.ts +1248 -0
- package/src/__tests__/shared.test.ts +2005 -0
- package/src/__tests__/styles.test.ts +14 -0
- package/src/__tests__/theme-schema.test.ts +47 -0
- package/src/__tests__/theme.test.ts +554 -0
- package/src/cli.ts +582 -0
- package/src/commands/init.ts +92 -0
- package/src/commands/update.ts +444 -0
- package/src/engine.ts +20 -0
- package/src/logger.ts +15 -0
- package/src/migrations/0000_schema_versioning.ts +67 -0
- package/src/migrations/0001_server_port.ts +52 -0
- package/src/migrations/0002_brand_logo.ts +49 -0
- package/src/migrations/index.ts +26 -0
- package/src/migrations/schema.ts +24 -0
- package/src/server-registry.ts +405 -0
- package/src/shared.ts +1239 -0
- package/src/site/styles.css +931 -0
- package/src/site/template.html +193 -0
- package/src/templates/crucible.ts +1163 -0
- package/src/templates/driver.ts +876 -0
- package/src/templates/handbook.ts +339 -0
- package/src/templates/minimal.ts +139 -0
- package/src/templates/pipeline.ts +966 -0
- package/src/templates/productbook.ts +1032 -0
- package/src/templates/runbook.ts +829 -0
- package/src/templates/schema.ts +119 -0
- package/src/templates/servicebook.ts +1242 -0
- package/src/theme.ts +245 -0
|
@@ -0,0 +1,966 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pipeline Template
|
|
3
|
+
*
|
|
4
|
+
* Extends minimal with structured sections for data pipeline operations.
|
|
5
|
+
* Sections: Pipeline, Sources, Destinations, Operations, Troubleshooting, Reference
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import type { TemplateContext, TemplateDef } from "./schema.ts";
|
|
9
|
+
|
|
10
|
+
export const pipeline: TemplateDef = {
|
|
11
|
+
id: "pipeline",
|
|
12
|
+
name: "Pipeline",
|
|
13
|
+
description: "Data pipeline operations with stages, sources, destinations, and manifests",
|
|
14
|
+
version: 1,
|
|
15
|
+
extends: "minimal",
|
|
16
|
+
sections: [
|
|
17
|
+
{
|
|
18
|
+
name: "Pipeline",
|
|
19
|
+
path: "content/pipeline",
|
|
20
|
+
description: "Pipeline stage documentation — the core dataflow",
|
|
21
|
+
},
|
|
22
|
+
{
|
|
23
|
+
name: "Sources",
|
|
24
|
+
path: "content/sources",
|
|
25
|
+
description: "Source data systems, schemas, auth profiles",
|
|
26
|
+
},
|
|
27
|
+
{
|
|
28
|
+
name: "Destinations",
|
|
29
|
+
path: "content/destinations",
|
|
30
|
+
description: "Target structures, layout conventions",
|
|
31
|
+
},
|
|
32
|
+
{
|
|
33
|
+
name: "Operations",
|
|
34
|
+
path: "content/operations",
|
|
35
|
+
description: "Run procedures, schedules, checklists",
|
|
36
|
+
},
|
|
37
|
+
{
|
|
38
|
+
name: "Troubleshooting",
|
|
39
|
+
path: "content/troubleshooting",
|
|
40
|
+
description: "Problem → solution guides",
|
|
41
|
+
},
|
|
42
|
+
{
|
|
43
|
+
name: "Reference",
|
|
44
|
+
path: "content/reference",
|
|
45
|
+
description: "Manifests, field mappings, metrics, contacts",
|
|
46
|
+
},
|
|
47
|
+
],
|
|
48
|
+
files: [
|
|
49
|
+
{
|
|
50
|
+
path: "site.yaml",
|
|
51
|
+
content: (ctx: TemplateContext) => `# ${ctx.branding.siteName} - Site Configuration
|
|
52
|
+
# Documentation: https://github.com/3leaps/kitfly
|
|
53
|
+
|
|
54
|
+
title: "${ctx.branding.siteName}"
|
|
55
|
+
|
|
56
|
+
# ← CUSTOMIZE: Your brand settings
|
|
57
|
+
brand:
|
|
58
|
+
name: "${ctx.branding.brandName}"
|
|
59
|
+
url: "${ctx.branding.brandUrl}"
|
|
60
|
+
# external: false # Set true if brand URL is external
|
|
61
|
+
|
|
62
|
+
# Content sections
|
|
63
|
+
sections:
|
|
64
|
+
- name: "Pipeline"
|
|
65
|
+
path: "content/pipeline"
|
|
66
|
+
- name: "Sources"
|
|
67
|
+
path: "content/sources"
|
|
68
|
+
- name: "Destinations"
|
|
69
|
+
path: "content/destinations"
|
|
70
|
+
- name: "Operations"
|
|
71
|
+
path: "content/operations"
|
|
72
|
+
- name: "Troubleshooting"
|
|
73
|
+
path: "content/troubleshooting"
|
|
74
|
+
- name: "Reference"
|
|
75
|
+
path: "content/reference"
|
|
76
|
+
|
|
77
|
+
# Home page
|
|
78
|
+
home: "index.md"
|
|
79
|
+
`,
|
|
80
|
+
},
|
|
81
|
+
{
|
|
82
|
+
path: "index.md",
|
|
83
|
+
content: (ctx: TemplateContext) => `---
|
|
84
|
+
title: Home
|
|
85
|
+
description: ${ctx.branding.siteName} - Data Pipeline Operations
|
|
86
|
+
---
|
|
87
|
+
|
|
88
|
+
# ${ctx.branding.siteName}
|
|
89
|
+
|
|
90
|
+
Data pipeline operations for ${ctx.branding.brandName}.
|
|
91
|
+
|
|
92
|
+
## Pipeline Status
|
|
93
|
+
|
|
94
|
+
| Stage | Description | Status |
|
|
95
|
+
|-------|-------------|--------|
|
|
96
|
+
| Index | Build searchable catalog of source data | — |
|
|
97
|
+
| Extract | Derive routing fields from content | — |
|
|
98
|
+
| Transfer | Copy/reorganize to destination structure | — |
|
|
99
|
+
| Validate | Verify completeness and correctness | — |
|
|
100
|
+
|
|
101
|
+
## Quick Links
|
|
102
|
+
|
|
103
|
+
### [Pipeline Overview](/content/pipeline/overview)
|
|
104
|
+
End-to-end dataflow: stages, dependencies, and execution order.
|
|
105
|
+
|
|
106
|
+
### [Run a Pipeline](/content/operations/run-pipeline)
|
|
107
|
+
Step-by-step execution procedure with checkpoints.
|
|
108
|
+
|
|
109
|
+
### [Source Systems](/content/sources/)
|
|
110
|
+
Source data systems, schemas, and authentication profiles.
|
|
111
|
+
|
|
112
|
+
### [Troubleshooting](/content/troubleshooting/common-issues)
|
|
113
|
+
Diagnose and resolve common pipeline issues.
|
|
114
|
+
|
|
115
|
+
---
|
|
116
|
+
|
|
117
|
+
*Last updated: ${new Date().toISOString().split("T")[0]}*
|
|
118
|
+
`,
|
|
119
|
+
},
|
|
120
|
+
// Pipeline section
|
|
121
|
+
{
|
|
122
|
+
path: "content/pipeline/overview.md",
|
|
123
|
+
content: (ctx: TemplateContext) => `---
|
|
124
|
+
title: Pipeline Overview
|
|
125
|
+
description: End-to-end dataflow for ${ctx.branding.brandName}
|
|
126
|
+
---
|
|
127
|
+
|
|
128
|
+
# Pipeline Overview
|
|
129
|
+
|
|
130
|
+
<!-- ← CUSTOMIZE: Describe your specific pipeline -->
|
|
131
|
+
|
|
132
|
+
## Dataflow
|
|
133
|
+
|
|
134
|
+
Source → Index → Extract → Transfer → Validate → Destination
|
|
135
|
+
|
|
136
|
+
## Stages
|
|
137
|
+
|
|
138
|
+
### 1. Index Build
|
|
139
|
+
|
|
140
|
+
Build searchable catalog of source objects for efficient querying.
|
|
141
|
+
|
|
142
|
+
- Input: Source bucket/path scope
|
|
143
|
+
- Output: Index database with object metadata
|
|
144
|
+
- See: [Index Build](/content/pipeline/index-build)
|
|
145
|
+
|
|
146
|
+
### 2. Content Extraction
|
|
147
|
+
|
|
148
|
+
Inspect file content to derive routing fields (dates, IDs, categories).
|
|
149
|
+
|
|
150
|
+
- Input: Indexed objects
|
|
151
|
+
- Output: Enriched metadata with extracted fields
|
|
152
|
+
- See: [Content Extraction](/content/pipeline/extract)
|
|
153
|
+
|
|
154
|
+
### 3. Transfer / Reflow
|
|
155
|
+
|
|
156
|
+
Copy objects from source to destination with key rewriting based on extracted fields.
|
|
157
|
+
|
|
158
|
+
- Input: Enriched index with routing decisions
|
|
159
|
+
- Output: Files in destination structure
|
|
160
|
+
- See: [Transfer](/content/pipeline/transfer)
|
|
161
|
+
|
|
162
|
+
### 4. Validation
|
|
163
|
+
|
|
164
|
+
Verify completeness, deduplication, and data quality.
|
|
165
|
+
|
|
166
|
+
- Input: Destination state + source index
|
|
167
|
+
- Output: Validation report
|
|
168
|
+
- See: [Validation](/content/pipeline/validate)
|
|
169
|
+
`,
|
|
170
|
+
},
|
|
171
|
+
{
|
|
172
|
+
path: "content/pipeline/index-build.md",
|
|
173
|
+
content: (_ctx: TemplateContext) => `---
|
|
174
|
+
title: Index Build
|
|
175
|
+
description: Build searchable index of source objects
|
|
176
|
+
---
|
|
177
|
+
|
|
178
|
+
# Index Build
|
|
179
|
+
|
|
180
|
+
<!-- ← CUSTOMIZE: Replace with your index build process -->
|
|
181
|
+
|
|
182
|
+
## Objective
|
|
183
|
+
|
|
184
|
+
Build a searchable index of source objects for a defined scope.
|
|
185
|
+
|
|
186
|
+
## Prerequisites
|
|
187
|
+
|
|
188
|
+
- [ ] Source credentials configured
|
|
189
|
+
- [ ] Scope defined (sites, date range)
|
|
190
|
+
- [ ] Manifest prepared
|
|
191
|
+
|
|
192
|
+
## Procedure
|
|
193
|
+
|
|
194
|
+
1. Prepare manifest YAML
|
|
195
|
+
2. Run index build
|
|
196
|
+
3. Verify object count
|
|
197
|
+
4. Query to validate coverage
|
|
198
|
+
|
|
199
|
+
## Manifests
|
|
200
|
+
|
|
201
|
+
See [Manifest Templates](/content/reference/manifests/) for examples.
|
|
202
|
+
|
|
203
|
+
## Verification
|
|
204
|
+
|
|
205
|
+
\`\`\`bash
|
|
206
|
+
# Check index count
|
|
207
|
+
# ← CUSTOMIZE: your verification command
|
|
208
|
+
\`\`\`
|
|
209
|
+
|
|
210
|
+
Expected: object count matches source scope.
|
|
211
|
+
`,
|
|
212
|
+
},
|
|
213
|
+
{
|
|
214
|
+
path: "content/pipeline/extract.md",
|
|
215
|
+
content: (_ctx: TemplateContext) => `---
|
|
216
|
+
title: Content Extraction
|
|
217
|
+
description: Derive routing fields from file content
|
|
218
|
+
---
|
|
219
|
+
|
|
220
|
+
# Content Extraction
|
|
221
|
+
|
|
222
|
+
<!-- ← CUSTOMIZE: Replace with your extraction rules -->
|
|
223
|
+
|
|
224
|
+
## Objective
|
|
225
|
+
|
|
226
|
+
Derive routing fields from file content to enable correct placement.
|
|
227
|
+
|
|
228
|
+
## Configuration
|
|
229
|
+
|
|
230
|
+
<!-- Extraction rules: XPath, regex, JSON path -->
|
|
231
|
+
|
|
232
|
+
| Field | Source | Extraction Method |
|
|
233
|
+
|-------|--------|-------------------|
|
|
234
|
+
| date | File content | Regex / XPath |
|
|
235
|
+
| entity_id | Filename or content | Pattern match |
|
|
236
|
+
| category | Content analysis | Rules engine |
|
|
237
|
+
|
|
238
|
+
## Procedure
|
|
239
|
+
|
|
240
|
+
1. Prepare extraction config
|
|
241
|
+
2. Run content probe
|
|
242
|
+
3. Verify extracted fields
|
|
243
|
+
4. Feed results to transfer stage
|
|
244
|
+
|
|
245
|
+
## Verification
|
|
246
|
+
|
|
247
|
+
\`\`\`bash
|
|
248
|
+
# Spot-check extracted fields
|
|
249
|
+
# ← CUSTOMIZE: your verification command
|
|
250
|
+
\`\`\`
|
|
251
|
+
`,
|
|
252
|
+
},
|
|
253
|
+
{
|
|
254
|
+
path: "content/pipeline/transfer.md",
|
|
255
|
+
content: (_ctx: TemplateContext) => `---
|
|
256
|
+
title: Transfer / Reflow
|
|
257
|
+
description: Copy objects from source to destination with key rewriting
|
|
258
|
+
---
|
|
259
|
+
|
|
260
|
+
# Transfer / Reflow
|
|
261
|
+
|
|
262
|
+
<!-- ← CUSTOMIZE: Replace with your transfer configuration -->
|
|
263
|
+
|
|
264
|
+
## Objective
|
|
265
|
+
|
|
266
|
+
Copy objects from source to destination with key rewriting.
|
|
267
|
+
|
|
268
|
+
## Configuration
|
|
269
|
+
|
|
270
|
+
- Source and destination profiles
|
|
271
|
+
- Rewrite templates
|
|
272
|
+
- Collision policy
|
|
273
|
+
- Checkpoint/resume settings
|
|
274
|
+
|
|
275
|
+
## Procedure
|
|
276
|
+
|
|
277
|
+
1. Prepare reflow command
|
|
278
|
+
2. Dry-run first
|
|
279
|
+
3. Execute with checkpoint
|
|
280
|
+
4. Verify landing structure
|
|
281
|
+
|
|
282
|
+
## Dry Run
|
|
283
|
+
|
|
284
|
+
\`\`\`bash
|
|
285
|
+
# ← CUSTOMIZE: your dry-run command
|
|
286
|
+
\`\`\`
|
|
287
|
+
|
|
288
|
+
**Verify**: Output shows expected file count and destination paths.
|
|
289
|
+
|
|
290
|
+
## Execute
|
|
291
|
+
|
|
292
|
+
\`\`\`bash
|
|
293
|
+
# ← CUSTOMIZE: your transfer command with checkpoint
|
|
294
|
+
\`\`\`
|
|
295
|
+
|
|
296
|
+
## Resume from Checkpoint
|
|
297
|
+
|
|
298
|
+
If interrupted:
|
|
299
|
+
|
|
300
|
+
\`\`\`bash
|
|
301
|
+
# ← CUSTOMIZE: your resume command
|
|
302
|
+
\`\`\`
|
|
303
|
+
`,
|
|
304
|
+
},
|
|
305
|
+
{
|
|
306
|
+
path: "content/pipeline/validate.md",
|
|
307
|
+
content: (_ctx: TemplateContext) => `---
|
|
308
|
+
title: Validation
|
|
309
|
+
description: Verify pipeline output is complete and correct
|
|
310
|
+
---
|
|
311
|
+
|
|
312
|
+
# Validation
|
|
313
|
+
|
|
314
|
+
<!-- ← CUSTOMIZE: Replace with your validation checks -->
|
|
315
|
+
|
|
316
|
+
## Objective
|
|
317
|
+
|
|
318
|
+
Verify pipeline output is complete and correct.
|
|
319
|
+
|
|
320
|
+
## Checks
|
|
321
|
+
|
|
322
|
+
- [ ] Object count matches expected
|
|
323
|
+
- [ ] No duplicate files in destination
|
|
324
|
+
- [ ] Routing fields match destination paths
|
|
325
|
+
- [ ] No error records in output
|
|
326
|
+
|
|
327
|
+
## Procedure
|
|
328
|
+
|
|
329
|
+
1. Run validation report
|
|
330
|
+
2. Compare source count vs destination count
|
|
331
|
+
3. Check for duplicates
|
|
332
|
+
4. Verify sample of routing decisions
|
|
333
|
+
|
|
334
|
+
## Validation Report
|
|
335
|
+
|
|
336
|
+
\`\`\`bash
|
|
337
|
+
# ← CUSTOMIZE: your validation command
|
|
338
|
+
\`\`\`
|
|
339
|
+
|
|
340
|
+
## Common Issues
|
|
341
|
+
|
|
342
|
+
| Issue | Cause | Resolution |
|
|
343
|
+
|-------|-------|------------|
|
|
344
|
+
| Count mismatch | Filtered items or errors | Check error log |
|
|
345
|
+
| Duplicates | Collision policy | Review collision settings |
|
|
346
|
+
| Wrong paths | Extraction rules | Verify field mappings |
|
|
347
|
+
`,
|
|
348
|
+
},
|
|
349
|
+
// Sources section
|
|
350
|
+
{
|
|
351
|
+
path: "content/sources/index.md",
|
|
352
|
+
content: (ctx: TemplateContext) => `---
|
|
353
|
+
title: Source Systems
|
|
354
|
+
description: Source data systems for ${ctx.branding.brandName}
|
|
355
|
+
---
|
|
356
|
+
|
|
357
|
+
# Source Systems
|
|
358
|
+
|
|
359
|
+
<!-- ← CUSTOMIZE: Add your source systems -->
|
|
360
|
+
|
|
361
|
+
| Source | Type | Auth | Documentation |
|
|
362
|
+
|--------|------|------|---------------|
|
|
363
|
+
| *source-name* | S3 / GCS / file | Profile / keys | [Details](./source-name) |
|
|
364
|
+
|
|
365
|
+
## Adding a New Source
|
|
366
|
+
|
|
367
|
+
1. Create a file in \`content/sources/\` describing the system
|
|
368
|
+
2. Document: connection details, auth profile, data format, schema
|
|
369
|
+
3. Update this index table
|
|
370
|
+
`,
|
|
371
|
+
},
|
|
372
|
+
// Destinations section
|
|
373
|
+
{
|
|
374
|
+
path: "content/destinations/index.md",
|
|
375
|
+
content: (ctx: TemplateContext) => `---
|
|
376
|
+
title: Destinations
|
|
377
|
+
description: Target structures for ${ctx.branding.brandName}
|
|
378
|
+
---
|
|
379
|
+
|
|
380
|
+
# Destinations
|
|
381
|
+
|
|
382
|
+
<!-- ← CUSTOMIZE: Add your destination systems -->
|
|
383
|
+
|
|
384
|
+
| Destination | Type | Structure | Documentation |
|
|
385
|
+
|-------------|------|-----------|---------------|
|
|
386
|
+
| *dest-name* | S3 / GCS / file | date-first / entity-first | [Details](./dest-name) |
|
|
387
|
+
|
|
388
|
+
## Path Structure
|
|
389
|
+
|
|
390
|
+
<!-- ← CUSTOMIZE: Document your target path layout -->
|
|
391
|
+
|
|
392
|
+
\`\`\`
|
|
393
|
+
destination-root/
|
|
394
|
+
├── {year}/
|
|
395
|
+
│ ├── {month}/
|
|
396
|
+
│ │ ├── {entity-id}/
|
|
397
|
+
│ │ │ └── {filename}
|
|
398
|
+
\`\`\`
|
|
399
|
+
|
|
400
|
+
## Adding a New Destination
|
|
401
|
+
|
|
402
|
+
1. Create a file in \`content/destinations/\` describing the target
|
|
403
|
+
2. Document: path structure, naming conventions, access controls
|
|
404
|
+
3. Update this index table
|
|
405
|
+
`,
|
|
406
|
+
},
|
|
407
|
+
// Operations section
|
|
408
|
+
{
|
|
409
|
+
path: "content/operations/run-pipeline.md",
|
|
410
|
+
content: (_ctx: TemplateContext) => `---
|
|
411
|
+
title: Run a Pipeline
|
|
412
|
+
description: Full pipeline execution procedure
|
|
413
|
+
---
|
|
414
|
+
|
|
415
|
+
# Run a Pipeline
|
|
416
|
+
|
|
417
|
+
<!-- ← CUSTOMIZE: Replace with your pipeline commands -->
|
|
418
|
+
|
|
419
|
+
## Prerequisites
|
|
420
|
+
|
|
421
|
+
- [ ] Source and destination credentials valid
|
|
422
|
+
- [ ] Index up to date for target scope
|
|
423
|
+
- [ ] Extraction config tested
|
|
424
|
+
- [ ] Dry-run completed
|
|
425
|
+
|
|
426
|
+
## Full Pipeline Execution
|
|
427
|
+
|
|
428
|
+
### Step 1: Index Build
|
|
429
|
+
|
|
430
|
+
\`\`\`bash
|
|
431
|
+
# ← CUSTOMIZE: your index build command
|
|
432
|
+
\`\`\`
|
|
433
|
+
|
|
434
|
+
**Verify**: Index count matches expected scope.
|
|
435
|
+
|
|
436
|
+
### Step 2: Content Probe
|
|
437
|
+
|
|
438
|
+
\`\`\`bash
|
|
439
|
+
# ← CUSTOMIZE: your extraction command
|
|
440
|
+
\`\`\`
|
|
441
|
+
|
|
442
|
+
**Verify**: Extracted fields look correct for sample objects.
|
|
443
|
+
|
|
444
|
+
### Step 3: Transfer Reflow
|
|
445
|
+
|
|
446
|
+
\`\`\`bash
|
|
447
|
+
# ← CUSTOMIZE: your transfer command
|
|
448
|
+
\`\`\`
|
|
449
|
+
|
|
450
|
+
**Verify**: Destination file count and structure.
|
|
451
|
+
|
|
452
|
+
### Step 4: Validate
|
|
453
|
+
|
|
454
|
+
\`\`\`bash
|
|
455
|
+
# ← CUSTOMIZE: your validation command
|
|
456
|
+
\`\`\`
|
|
457
|
+
|
|
458
|
+
**Verify**: Validation report shows no errors.
|
|
459
|
+
|
|
460
|
+
## Checkpoint / Resume
|
|
461
|
+
|
|
462
|
+
If interrupted, resume from checkpoint:
|
|
463
|
+
|
|
464
|
+
\`\`\`bash
|
|
465
|
+
# ← CUSTOMIZE: your resume command
|
|
466
|
+
\`\`\`
|
|
467
|
+
|
|
468
|
+
## Post-Run
|
|
469
|
+
|
|
470
|
+
- [ ] Review validation report
|
|
471
|
+
- [ ] Update pipeline status table in [Home](/)
|
|
472
|
+
- [ ] Log run in [Schedules](/content/operations/schedules)
|
|
473
|
+
`,
|
|
474
|
+
},
|
|
475
|
+
{
|
|
476
|
+
path: "content/operations/schedules.md",
|
|
477
|
+
content: (_ctx: TemplateContext) => `---
|
|
478
|
+
title: Schedules
|
|
479
|
+
description: Pipeline execution schedules
|
|
480
|
+
---
|
|
481
|
+
|
|
482
|
+
# Schedules
|
|
483
|
+
|
|
484
|
+
<!-- ← CUSTOMIZE: Add your pipeline schedules -->
|
|
485
|
+
|
|
486
|
+
| Pipeline | Frequency | Scope | Notes |
|
|
487
|
+
|----------|-----------|-------|-------|
|
|
488
|
+
| *pipeline-name* | Daily / Weekly / Ad hoc | *scope* | *notes* |
|
|
489
|
+
|
|
490
|
+
## Schedule Management
|
|
491
|
+
|
|
492
|
+
- Automated pipelines run via cron/scheduler
|
|
493
|
+
- Ad hoc runs documented here after execution
|
|
494
|
+
- See [Run a Pipeline](/content/operations/run-pipeline) for execution steps
|
|
495
|
+
`,
|
|
496
|
+
},
|
|
497
|
+
// Troubleshooting section
|
|
498
|
+
{
|
|
499
|
+
path: "content/troubleshooting/common-issues.md",
|
|
500
|
+
content: (_ctx: TemplateContext) => `---
|
|
501
|
+
title: Common Issues
|
|
502
|
+
description: Troubleshooting guide for pipeline problems
|
|
503
|
+
---
|
|
504
|
+
|
|
505
|
+
# Common Issues
|
|
506
|
+
|
|
507
|
+
<!-- ← CUSTOMIZE: Add your pipeline-specific issues -->
|
|
508
|
+
|
|
509
|
+
## Authentication Expiry Mid-Pipeline
|
|
510
|
+
|
|
511
|
+
**Symptoms**:
|
|
512
|
+
- Pipeline fails partway through with auth errors
|
|
513
|
+
- "Access Denied" or "Token expired" in logs
|
|
514
|
+
- Partial results in destination
|
|
515
|
+
|
|
516
|
+
**Possible Causes**:
|
|
517
|
+
1. Source credentials expired during long-running job
|
|
518
|
+
2. Token refresh not configured
|
|
519
|
+
3. Session timeout shorter than pipeline duration
|
|
520
|
+
|
|
521
|
+
**Resolution**:
|
|
522
|
+
|
|
523
|
+
1. Check credential expiry:
|
|
524
|
+
\`\`\`bash
|
|
525
|
+
# ← CUSTOMIZE: your credential check command
|
|
526
|
+
\`\`\`
|
|
527
|
+
|
|
528
|
+
2. Refresh credentials and resume from checkpoint
|
|
529
|
+
3. Consider longer-lived tokens for large scopes
|
|
530
|
+
|
|
531
|
+
**Prevention**: Configure token refresh or use service account credentials.
|
|
532
|
+
|
|
533
|
+
---
|
|
534
|
+
|
|
535
|
+
## Index Lock / Corruption
|
|
536
|
+
|
|
537
|
+
**Symptoms**:
|
|
538
|
+
- "Index locked" or "Database is locked" errors
|
|
539
|
+
- Index build hangs or returns stale data
|
|
540
|
+
- Concurrent pipeline runs fail
|
|
541
|
+
|
|
542
|
+
**Resolution**:
|
|
543
|
+
|
|
544
|
+
1. Check for other running pipeline processes
|
|
545
|
+
2. Release lock if safe:
|
|
546
|
+
\`\`\`bash
|
|
547
|
+
# ← CUSTOMIZE: your lock release command
|
|
548
|
+
\`\`\`
|
|
549
|
+
3. If corrupted, rebuild index from scratch
|
|
550
|
+
|
|
551
|
+
---
|
|
552
|
+
|
|
553
|
+
## Duplicate Detection Conflicts
|
|
554
|
+
|
|
555
|
+
**Symptoms**:
|
|
556
|
+
- Validation reports unexpected duplicates
|
|
557
|
+
- Same file appears in multiple destination paths
|
|
558
|
+
- Object count exceeds expected
|
|
559
|
+
|
|
560
|
+
**Resolution**:
|
|
561
|
+
|
|
562
|
+
1. Review extraction rules — are routing fields unique?
|
|
563
|
+
2. Check collision policy (overwrite vs skip vs error)
|
|
564
|
+
3. Inspect duplicate pairs to find root cause
|
|
565
|
+
|
|
566
|
+
---
|
|
567
|
+
|
|
568
|
+
## Destination Collision Handling
|
|
569
|
+
|
|
570
|
+
**Symptoms**:
|
|
571
|
+
- Files overwritten unexpectedly
|
|
572
|
+
- "Collision" warnings in transfer log
|
|
573
|
+
- Missing files in destination
|
|
574
|
+
|
|
575
|
+
**Resolution**:
|
|
576
|
+
|
|
577
|
+
1. Review collision policy setting
|
|
578
|
+
2. Check if source contains genuine duplicates
|
|
579
|
+
3. Adjust rewrite template to produce unique keys
|
|
580
|
+
`,
|
|
581
|
+
},
|
|
582
|
+
// Reference section subdirectories
|
|
583
|
+
{
|
|
584
|
+
path: "content/reference/manifests/job-template.md",
|
|
585
|
+
content: (_ctx: TemplateContext) => `---
|
|
586
|
+
title: Job Manifest Template
|
|
587
|
+
description: Template for pipeline job manifests
|
|
588
|
+
---
|
|
589
|
+
|
|
590
|
+
# Job Manifest Template
|
|
591
|
+
|
|
592
|
+
<!-- ← CUSTOMIZE: Replace with your manifest format -->
|
|
593
|
+
|
|
594
|
+
\`\`\`yaml
|
|
595
|
+
# Pipeline Job Manifest
|
|
596
|
+
job:
|
|
597
|
+
name: "descriptive-job-name"
|
|
598
|
+
pipeline: "pipeline-id"
|
|
599
|
+
scope:
|
|
600
|
+
# ← CUSTOMIZE: define your scope fields
|
|
601
|
+
sites: []
|
|
602
|
+
date_range:
|
|
603
|
+
start: "YYYY-MM-DD"
|
|
604
|
+
end: "YYYY-MM-DD"
|
|
605
|
+
source:
|
|
606
|
+
profile: "source-profile-name"
|
|
607
|
+
destination:
|
|
608
|
+
profile: "dest-profile-name"
|
|
609
|
+
options:
|
|
610
|
+
dry_run: false
|
|
611
|
+
checkpoint: true
|
|
612
|
+
collision: "skip" # skip | overwrite | error
|
|
613
|
+
\`\`\`
|
|
614
|
+
|
|
615
|
+
## Fields
|
|
616
|
+
|
|
617
|
+
| Field | Required | Description |
|
|
618
|
+
|-------|----------|-------------|
|
|
619
|
+
| \`job.name\` | Yes | Human-readable job identifier |
|
|
620
|
+
| \`job.pipeline\` | Yes | Pipeline definition to execute |
|
|
621
|
+
| \`job.scope\` | Yes | What data to process |
|
|
622
|
+
| \`job.source.profile\` | Yes | Source credential/connection profile |
|
|
623
|
+
| \`job.destination.profile\` | Yes | Destination credential/connection profile |
|
|
624
|
+
| \`job.options.dry_run\` | No | Preview without writing (default: false) |
|
|
625
|
+
| \`job.options.checkpoint\` | No | Enable resume capability (default: true) |
|
|
626
|
+
| \`job.options.collision\` | No | How to handle existing files (default: skip) |
|
|
627
|
+
`,
|
|
628
|
+
},
|
|
629
|
+
{
|
|
630
|
+
path: "content/reference/field-mappings/mapping-template.md",
|
|
631
|
+
content: (_ctx: TemplateContext) => `---
|
|
632
|
+
title: Field Mapping Template
|
|
633
|
+
description: Source field → destination field documentation
|
|
634
|
+
---
|
|
635
|
+
|
|
636
|
+
# Field Mapping: [Source System]
|
|
637
|
+
|
|
638
|
+
<!-- ← CUSTOMIZE: Document your field mappings -->
|
|
639
|
+
|
|
640
|
+
## Overview
|
|
641
|
+
|
|
642
|
+
| Source Field | Destination Field | Transform | Notes |
|
|
643
|
+
|-------------|-------------------|-----------|-------|
|
|
644
|
+
| \`source.field_a\` | \`dest.field_x\` | Direct copy | |
|
|
645
|
+
| \`source.field_b\` | \`dest.field_y\` | Date format | YYYY-MM-DD → YYYYMMDD |
|
|
646
|
+
| \`source.field_c\` | \`dest.field_z\` | Lookup table | See mapping table below |
|
|
647
|
+
|
|
648
|
+
## Transform Rules
|
|
649
|
+
|
|
650
|
+
### Date Formatting
|
|
651
|
+
|
|
652
|
+
Input: \`2024-03-15\` → Output: \`20240315\`
|
|
653
|
+
|
|
654
|
+
### Lookup Tables
|
|
655
|
+
|
|
656
|
+
| Source Value | Destination Value |
|
|
657
|
+
|-------------|-------------------|
|
|
658
|
+
| \`type_a\` | \`A\` |
|
|
659
|
+
| \`type_b\` | \`B\` |
|
|
660
|
+
`,
|
|
661
|
+
},
|
|
662
|
+
{
|
|
663
|
+
path: "content/reference/metrics/pipeline-kpis.md",
|
|
664
|
+
content: (_ctx: TemplateContext) => `---
|
|
665
|
+
title: Pipeline KPIs
|
|
666
|
+
description: Key performance indicators for pipeline operations
|
|
667
|
+
---
|
|
668
|
+
|
|
669
|
+
# Pipeline KPIs
|
|
670
|
+
|
|
671
|
+
<!-- ← CUSTOMIZE: Add your metrics and dashboard links -->
|
|
672
|
+
|
|
673
|
+
## Key Metrics
|
|
674
|
+
|
|
675
|
+
| Metric | Target | Dashboard |
|
|
676
|
+
|--------|--------|-----------|
|
|
677
|
+
| Pipeline success rate | > 99% | [Link] |
|
|
678
|
+
| Average run duration | < X hours | [Link] |
|
|
679
|
+
| Object throughput | > X/hour | [Link] |
|
|
680
|
+
| Error rate | < 0.1% | [Link] |
|
|
681
|
+
|
|
682
|
+
## SLAs
|
|
683
|
+
|
|
684
|
+
| Pipeline | Frequency | Completion Window | Escalation |
|
|
685
|
+
|----------|-----------|-------------------|------------|
|
|
686
|
+
| *pipeline-name* | Daily | 4 hours | Page on-call |
|
|
687
|
+
|
|
688
|
+
## Alerting
|
|
689
|
+
|
|
690
|
+
| Alert | Warning | Critical | Action |
|
|
691
|
+
|-------|---------|----------|--------|
|
|
692
|
+
| Run duration | > 2x average | > 4x average | Investigate |
|
|
693
|
+
| Error count | > 10 | > 100 | Check logs |
|
|
694
|
+
| Stale index | > 24h | > 48h | Rebuild |
|
|
695
|
+
`,
|
|
696
|
+
},
|
|
697
|
+
{
|
|
698
|
+
path: "content/reference/checklists/pre-run.md",
|
|
699
|
+
content: (_ctx: TemplateContext) => `---
|
|
700
|
+
title: Pre-Run Checklist
|
|
701
|
+
description: Verification checklist before running a pipeline
|
|
702
|
+
---
|
|
703
|
+
|
|
704
|
+
# Pre-Run Checklist
|
|
705
|
+
|
|
706
|
+
<!-- ← CUSTOMIZE: Adapt to your pipeline process -->
|
|
707
|
+
|
|
708
|
+
## Credentials
|
|
709
|
+
|
|
710
|
+
- [ ] Source credentials valid and not expiring soon
|
|
711
|
+
- [ ] Destination credentials valid
|
|
712
|
+
- [ ] Service account permissions verified
|
|
713
|
+
|
|
714
|
+
## Scope
|
|
715
|
+
|
|
716
|
+
- [ ] Scope definition reviewed (sites, date range)
|
|
717
|
+
- [ ] Expected object count estimated
|
|
718
|
+
- [ ] No overlapping concurrent runs
|
|
719
|
+
|
|
720
|
+
## Configuration
|
|
721
|
+
|
|
722
|
+
- [ ] Manifest YAML validated
|
|
723
|
+
- [ ] Extraction rules tested on sample
|
|
724
|
+
- [ ] Destination path template verified
|
|
725
|
+
- [ ] Collision policy appropriate for this run
|
|
726
|
+
|
|
727
|
+
## Environment
|
|
728
|
+
|
|
729
|
+
- [ ] Sufficient disk space for index
|
|
730
|
+
- [ ] Network connectivity to source and destination
|
|
731
|
+
- [ ] Monitoring and alerting configured
|
|
732
|
+
|
|
733
|
+
## Go/No-Go
|
|
734
|
+
|
|
735
|
+
| Criteria | Status |
|
|
736
|
+
|----------|--------|
|
|
737
|
+
| Credentials valid | ⬜ |
|
|
738
|
+
| Scope defined | ⬜ |
|
|
739
|
+
| Config tested | ⬜ |
|
|
740
|
+
| Dry-run passed | ⬜ |
|
|
741
|
+
|
|
742
|
+
**Decision**: ⬜ GO / ⬜ NO-GO
|
|
743
|
+
`,
|
|
744
|
+
},
|
|
745
|
+
{
|
|
746
|
+
path: "content/reference/contacts/directory.md",
|
|
747
|
+
content: (ctx: TemplateContext) => `---
|
|
748
|
+
title: Contact Directory
|
|
749
|
+
description: Team and vendor contacts for ${ctx.branding.brandName}
|
|
750
|
+
---
|
|
751
|
+
|
|
752
|
+
# Contact Directory
|
|
753
|
+
|
|
754
|
+
<!-- ← CUSTOMIZE: Add your team and vendor contacts -->
|
|
755
|
+
|
|
756
|
+
## Pipeline Team
|
|
757
|
+
|
|
758
|
+
| Role | Contact | Responsibility |
|
|
759
|
+
|------|---------|----------------|
|
|
760
|
+
| Pipeline Lead | [Name/Email] | Architecture, operations |
|
|
761
|
+
| Data Engineer | [Name/Email] | Source/destination config |
|
|
762
|
+
| On-Call | [PagerDuty/Slack] | Incident response |
|
|
763
|
+
|
|
764
|
+
## Vendor Contacts
|
|
765
|
+
|
|
766
|
+
| Vendor | Type | Contact |
|
|
767
|
+
|--------|------|---------|
|
|
768
|
+
| Cloud Provider | Support | [Portal/Email] |
|
|
769
|
+
| Data Source | Support | [Portal/Email] |
|
|
770
|
+
|
|
771
|
+
## Escalation
|
|
772
|
+
|
|
773
|
+
For pipeline failures during business hours, contact the Pipeline Lead.
|
|
774
|
+
For off-hours incidents, page on-call via PagerDuty.
|
|
775
|
+
`,
|
|
776
|
+
},
|
|
777
|
+
// CUSTOMIZING.md
|
|
778
|
+
{
|
|
779
|
+
path: "CUSTOMIZING.md",
|
|
780
|
+
content: (ctx: TemplateContext) => `---
|
|
781
|
+
template: pipeline
|
|
782
|
+
template_version: 1
|
|
783
|
+
created: ${new Date().toISOString().split("T")[0]}
|
|
784
|
+
---
|
|
785
|
+
|
|
786
|
+
# Customizing ${ctx.branding.siteName}
|
|
787
|
+
|
|
788
|
+
This guide helps you (and AI assistants) understand how to customize this pipeline operations site.
|
|
789
|
+
|
|
790
|
+
## Site Structure
|
|
791
|
+
|
|
792
|
+
\`\`\`
|
|
793
|
+
${ctx.name}/
|
|
794
|
+
├── site.yaml # Site configuration (sections, branding)
|
|
795
|
+
├── theme.yaml # Theme customization (create if needed)
|
|
796
|
+
├── index.md # Home page with pipeline status
|
|
797
|
+
├── CUSTOMIZING.md # This file
|
|
798
|
+
├── content/
|
|
799
|
+
│ ├── pipeline/ # Pipeline stage documentation
|
|
800
|
+
│ ├── sources/ # Source data systems
|
|
801
|
+
│ ├── destinations/ # Target structures
|
|
802
|
+
│ ├── operations/ # Run procedures, schedules
|
|
803
|
+
│ ├── troubleshooting/ # Problem → solution guides
|
|
804
|
+
│ └── reference/ # Supporting materials
|
|
805
|
+
│ ├── manifests/ # Job manifest templates
|
|
806
|
+
│ ├── field-mappings/ # Source → destination mappings
|
|
807
|
+
│ ├── metrics/ # KPIs, SLAs, dashboards
|
|
808
|
+
│ ├── checklists/ # Pre-run, post-run checklists
|
|
809
|
+
│ └── contacts/ # Team contacts, escalation
|
|
810
|
+
└── assets/
|
|
811
|
+
└── brand/ # Logo, favicon
|
|
812
|
+
\`\`\`
|
|
813
|
+
|
|
814
|
+
## Configuration Files
|
|
815
|
+
|
|
816
|
+
### site.yaml - Site Configuration
|
|
817
|
+
|
|
818
|
+
\`\`\`yaml
|
|
819
|
+
title: "${ctx.branding.siteName}"
|
|
820
|
+
|
|
821
|
+
brand:
|
|
822
|
+
name: "${ctx.branding.brandName}" # Shown in header
|
|
823
|
+
url: "/" # Logo link destination
|
|
824
|
+
|
|
825
|
+
sections:
|
|
826
|
+
- name: "Pipeline"
|
|
827
|
+
path: "content/pipeline"
|
|
828
|
+
# Add or modify sections here
|
|
829
|
+
\`\`\`
|
|
830
|
+
|
|
831
|
+
### theme.yaml - Visual Customization (optional)
|
|
832
|
+
|
|
833
|
+
Create \`theme.yaml\` to customize colors and typography:
|
|
834
|
+
|
|
835
|
+
\`\`\`yaml
|
|
836
|
+
colors:
|
|
837
|
+
primary: "#2563eb"
|
|
838
|
+
background: "#ffffff"
|
|
839
|
+
text: "#1f2937"
|
|
840
|
+
|
|
841
|
+
footer:
|
|
842
|
+
text: "© ${ctx.year} ${ctx.branding.brandName}"
|
|
843
|
+
\`\`\`
|
|
844
|
+
|
|
845
|
+
## Adding Content
|
|
846
|
+
|
|
847
|
+
### New Pipeline Stage
|
|
848
|
+
|
|
849
|
+
1. Create file in \`content/pipeline/\`:
|
|
850
|
+
\`\`\`
|
|
851
|
+
content/pipeline/my-stage.md
|
|
852
|
+
\`\`\`
|
|
853
|
+
|
|
854
|
+
2. Use the stage format:
|
|
855
|
+
\`\`\`markdown
|
|
856
|
+
---
|
|
857
|
+
title: Stage Name
|
|
858
|
+
---
|
|
859
|
+
|
|
860
|
+
# Stage Name
|
|
861
|
+
|
|
862
|
+
## Objective
|
|
863
|
+
What this stage accomplishes.
|
|
864
|
+
|
|
865
|
+
## Prerequisites
|
|
866
|
+
- [ ] Required items
|
|
867
|
+
|
|
868
|
+
## Procedure
|
|
869
|
+
1. Step with command
|
|
870
|
+
2. Verification step
|
|
871
|
+
|
|
872
|
+
## Verification
|
|
873
|
+
How to confirm the stage completed correctly.
|
|
874
|
+
\`\`\`
|
|
875
|
+
|
|
876
|
+
3. Update the overview in \`content/pipeline/overview.md\`
|
|
877
|
+
|
|
878
|
+
### New Source System
|
|
879
|
+
|
|
880
|
+
1. Create file in \`content/sources/\`:
|
|
881
|
+
\`\`\`
|
|
882
|
+
content/sources/my-source.md
|
|
883
|
+
\`\`\`
|
|
884
|
+
|
|
885
|
+
2. Document: connection details, auth profile, data format, schema
|
|
886
|
+
3. Update the index table in \`content/sources/index.md\`
|
|
887
|
+
|
|
888
|
+
### New Destination
|
|
889
|
+
|
|
890
|
+
1. Create file in \`content/destinations/\`:
|
|
891
|
+
\`\`\`
|
|
892
|
+
content/destinations/my-destination.md
|
|
893
|
+
\`\`\`
|
|
894
|
+
|
|
895
|
+
2. Document: path structure, naming conventions, access controls
|
|
896
|
+
3. Update the index table in \`content/destinations/index.md\`
|
|
897
|
+
|
|
898
|
+
### Creating Manifest Templates
|
|
899
|
+
|
|
900
|
+
1. Create in \`content/reference/manifests/\`:
|
|
901
|
+
\`\`\`
|
|
902
|
+
content/reference/manifests/my-job.md
|
|
903
|
+
\`\`\`
|
|
904
|
+
|
|
905
|
+
2. Include the full YAML template with field descriptions
|
|
906
|
+
3. Document required vs optional fields
|
|
907
|
+
|
|
908
|
+
### New Troubleshooting Guide
|
|
909
|
+
|
|
910
|
+
Follow the pattern: Symptoms → Causes → Resolution → Prevention
|
|
911
|
+
|
|
912
|
+
### New Section
|
|
913
|
+
|
|
914
|
+
1. Create folder: \`content/newsection/\`
|
|
915
|
+
2. Add to \`site.yaml\`:
|
|
916
|
+
\`\`\`yaml
|
|
917
|
+
sections:
|
|
918
|
+
- name: "New Section"
|
|
919
|
+
path: "content/newsection"
|
|
920
|
+
\`\`\`
|
|
921
|
+
3. Add at least one markdown file
|
|
922
|
+
|
|
923
|
+
## Linking and References
|
|
924
|
+
|
|
925
|
+
### Internal Links
|
|
926
|
+
|
|
927
|
+
\`\`\`markdown
|
|
928
|
+
See [Pipeline Overview](/content/pipeline/overview) for the full dataflow.
|
|
929
|
+
\`\`\`
|
|
930
|
+
|
|
931
|
+
### External Links
|
|
932
|
+
|
|
933
|
+
\`\`\`markdown
|
|
934
|
+
See [Vendor Documentation](https://vendor.com/docs).
|
|
935
|
+
\`\`\`
|
|
936
|
+
|
|
937
|
+
## Document Conventions
|
|
938
|
+
|
|
939
|
+
### Pipeline Stages
|
|
940
|
+
|
|
941
|
+
- Start with Objective (what/why)
|
|
942
|
+
- List Prerequisites as checkboxes
|
|
943
|
+
- Number steps with verification
|
|
944
|
+
- Include expected outputs
|
|
945
|
+
|
|
946
|
+
### Troubleshooting
|
|
947
|
+
|
|
948
|
+
- Lead with Symptoms (what user sees)
|
|
949
|
+
- List Possible Causes
|
|
950
|
+
- Provide step-by-step Resolution
|
|
951
|
+
- Include Prevention guidance
|
|
952
|
+
|
|
953
|
+
### Checklists
|
|
954
|
+
|
|
955
|
+
- Use checkbox format: \`- [ ] Item\`
|
|
956
|
+
- Group by phase or category
|
|
957
|
+
- Include Go/No-Go decision point
|
|
958
|
+
|
|
959
|
+
## Getting Help
|
|
960
|
+
|
|
961
|
+
- [Kitfly Documentation](https://github.com/3leaps/kitfly)
|
|
962
|
+
- [Markdown Guide](https://www.markdownguide.org/)
|
|
963
|
+
`,
|
|
964
|
+
},
|
|
965
|
+
],
|
|
966
|
+
};
|