kitfly 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. package/CHANGELOG.md +60 -0
  2. package/LICENSE +21 -0
  3. package/README.md +136 -0
  4. package/VERSION +1 -0
  5. package/package.json +63 -0
  6. package/schemas/README.md +32 -0
  7. package/schemas/site.schema.json +5 -0
  8. package/schemas/theme.schema.json +5 -0
  9. package/schemas/v0/site.schema.json +172 -0
  10. package/schemas/v0/theme.schema.json +210 -0
  11. package/scripts/build-all.ts +121 -0
  12. package/scripts/build.ts +601 -0
  13. package/scripts/bundle.ts +781 -0
  14. package/scripts/dev.ts +777 -0
  15. package/scripts/generate-checksums.sh +78 -0
  16. package/scripts/release/export-release-key.sh +28 -0
  17. package/scripts/release/release-guard-tag-version.sh +79 -0
  18. package/scripts/release/sign-release-assets.sh +123 -0
  19. package/scripts/release/upload-release-assets.sh +76 -0
  20. package/scripts/release/upload-release-provenance.sh +52 -0
  21. package/scripts/release/verify-public-key.sh +48 -0
  22. package/scripts/release/verify-signatures.sh +117 -0
  23. package/scripts/version-sync.ts +82 -0
  24. package/src/__tests__/build.test.ts +240 -0
  25. package/src/__tests__/bundle.test.ts +786 -0
  26. package/src/__tests__/cli.test.ts +706 -0
  27. package/src/__tests__/crucible.test.ts +1043 -0
  28. package/src/__tests__/engine.test.ts +157 -0
  29. package/src/__tests__/init.test.ts +450 -0
  30. package/src/__tests__/pipeline.test.ts +1087 -0
  31. package/src/__tests__/productbook.test.ts +1206 -0
  32. package/src/__tests__/runbook.test.ts +974 -0
  33. package/src/__tests__/server-registry.test.ts +1251 -0
  34. package/src/__tests__/servicebook.test.ts +1248 -0
  35. package/src/__tests__/shared.test.ts +2005 -0
  36. package/src/__tests__/styles.test.ts +14 -0
  37. package/src/__tests__/theme-schema.test.ts +47 -0
  38. package/src/__tests__/theme.test.ts +554 -0
  39. package/src/cli.ts +582 -0
  40. package/src/commands/init.ts +92 -0
  41. package/src/commands/update.ts +444 -0
  42. package/src/engine.ts +20 -0
  43. package/src/logger.ts +15 -0
  44. package/src/migrations/0000_schema_versioning.ts +67 -0
  45. package/src/migrations/0001_server_port.ts +52 -0
  46. package/src/migrations/0002_brand_logo.ts +49 -0
  47. package/src/migrations/index.ts +26 -0
  48. package/src/migrations/schema.ts +24 -0
  49. package/src/server-registry.ts +405 -0
  50. package/src/shared.ts +1239 -0
  51. package/src/site/styles.css +931 -0
  52. package/src/site/template.html +193 -0
  53. package/src/templates/crucible.ts +1163 -0
  54. package/src/templates/driver.ts +876 -0
  55. package/src/templates/handbook.ts +339 -0
  56. package/src/templates/minimal.ts +139 -0
  57. package/src/templates/pipeline.ts +966 -0
  58. package/src/templates/productbook.ts +1032 -0
  59. package/src/templates/runbook.ts +829 -0
  60. package/src/templates/schema.ts +119 -0
  61. package/src/templates/servicebook.ts +1242 -0
  62. package/src/theme.ts +245 -0
@@ -0,0 +1,966 @@
1
+ /**
2
+ * Pipeline Template
3
+ *
4
+ * Extends minimal with structured sections for data pipeline operations.
5
+ * Sections: Pipeline, Sources, Destinations, Operations, Troubleshooting, Reference
6
+ */
7
+
8
+ import type { TemplateContext, TemplateDef } from "./schema.ts";
9
+
10
+ export const pipeline: TemplateDef = {
11
+ id: "pipeline",
12
+ name: "Pipeline",
13
+ description: "Data pipeline operations with stages, sources, destinations, and manifests",
14
+ version: 1,
15
+ extends: "minimal",
16
+ sections: [
17
+ {
18
+ name: "Pipeline",
19
+ path: "content/pipeline",
20
+ description: "Pipeline stage documentation — the core dataflow",
21
+ },
22
+ {
23
+ name: "Sources",
24
+ path: "content/sources",
25
+ description: "Source data systems, schemas, auth profiles",
26
+ },
27
+ {
28
+ name: "Destinations",
29
+ path: "content/destinations",
30
+ description: "Target structures, layout conventions",
31
+ },
32
+ {
33
+ name: "Operations",
34
+ path: "content/operations",
35
+ description: "Run procedures, schedules, checklists",
36
+ },
37
+ {
38
+ name: "Troubleshooting",
39
+ path: "content/troubleshooting",
40
+ description: "Problem → solution guides",
41
+ },
42
+ {
43
+ name: "Reference",
44
+ path: "content/reference",
45
+ description: "Manifests, field mappings, metrics, contacts",
46
+ },
47
+ ],
48
+ files: [
49
+ {
50
+ path: "site.yaml",
51
+ content: (ctx: TemplateContext) => `# ${ctx.branding.siteName} - Site Configuration
52
+ # Documentation: https://github.com/3leaps/kitfly
53
+
54
+ title: "${ctx.branding.siteName}"
55
+
56
+ # ← CUSTOMIZE: Your brand settings
57
+ brand:
58
+ name: "${ctx.branding.brandName}"
59
+ url: "${ctx.branding.brandUrl}"
60
+ # external: false # Set true if brand URL is external
61
+
62
+ # Content sections
63
+ sections:
64
+ - name: "Pipeline"
65
+ path: "content/pipeline"
66
+ - name: "Sources"
67
+ path: "content/sources"
68
+ - name: "Destinations"
69
+ path: "content/destinations"
70
+ - name: "Operations"
71
+ path: "content/operations"
72
+ - name: "Troubleshooting"
73
+ path: "content/troubleshooting"
74
+ - name: "Reference"
75
+ path: "content/reference"
76
+
77
+ # Home page
78
+ home: "index.md"
79
+ `,
80
+ },
81
+ {
82
+ path: "index.md",
83
+ content: (ctx: TemplateContext) => `---
84
+ title: Home
85
+ description: ${ctx.branding.siteName} - Data Pipeline Operations
86
+ ---
87
+
88
+ # ${ctx.branding.siteName}
89
+
90
+ Data pipeline operations for ${ctx.branding.brandName}.
91
+
92
+ ## Pipeline Status
93
+
94
+ | Stage | Description | Status |
95
+ |-------|-------------|--------|
96
+ | Index | Build searchable catalog of source data | — |
97
+ | Extract | Derive routing fields from content | — |
98
+ | Transfer | Copy/reorganize to destination structure | — |
99
+ | Validate | Verify completeness and correctness | — |
100
+
101
+ ## Quick Links
102
+
103
+ ### [Pipeline Overview](/content/pipeline/overview)
104
+ End-to-end dataflow: stages, dependencies, and execution order.
105
+
106
+ ### [Run a Pipeline](/content/operations/run-pipeline)
107
+ Step-by-step execution procedure with checkpoints.
108
+
109
+ ### [Source Systems](/content/sources/)
110
+ Source data systems, schemas, and authentication profiles.
111
+
112
+ ### [Troubleshooting](/content/troubleshooting/common-issues)
113
+ Diagnose and resolve common pipeline issues.
114
+
115
+ ---
116
+
117
+ *Last updated: ${new Date().toISOString().split("T")[0]}*
118
+ `,
119
+ },
120
+ // Pipeline section
121
+ {
122
+ path: "content/pipeline/overview.md",
123
+ content: (ctx: TemplateContext) => `---
124
+ title: Pipeline Overview
125
+ description: End-to-end dataflow for ${ctx.branding.brandName}
126
+ ---
127
+
128
+ # Pipeline Overview
129
+
130
+ <!-- ← CUSTOMIZE: Describe your specific pipeline -->
131
+
132
+ ## Dataflow
133
+
134
+ Source → Index → Extract → Transfer → Validate → Destination
135
+
136
+ ## Stages
137
+
138
+ ### 1. Index Build
139
+
140
+ Build searchable catalog of source objects for efficient querying.
141
+
142
+ - Input: Source bucket/path scope
143
+ - Output: Index database with object metadata
144
+ - See: [Index Build](/content/pipeline/index-build)
145
+
146
+ ### 2. Content Extraction
147
+
148
+ Inspect file content to derive routing fields (dates, IDs, categories).
149
+
150
+ - Input: Indexed objects
151
+ - Output: Enriched metadata with extracted fields
152
+ - See: [Content Extraction](/content/pipeline/extract)
153
+
154
+ ### 3. Transfer / Reflow
155
+
156
+ Copy objects from source to destination with key rewriting based on extracted fields.
157
+
158
+ - Input: Enriched index with routing decisions
159
+ - Output: Files in destination structure
160
+ - See: [Transfer](/content/pipeline/transfer)
161
+
162
+ ### 4. Validation
163
+
164
+ Verify completeness, deduplication, and data quality.
165
+
166
+ - Input: Destination state + source index
167
+ - Output: Validation report
168
+ - See: [Validation](/content/pipeline/validate)
169
+ `,
170
+ },
171
+ {
172
+ path: "content/pipeline/index-build.md",
173
+ content: (_ctx: TemplateContext) => `---
174
+ title: Index Build
175
+ description: Build searchable index of source objects
176
+ ---
177
+
178
+ # Index Build
179
+
180
+ <!-- ← CUSTOMIZE: Replace with your index build process -->
181
+
182
+ ## Objective
183
+
184
+ Build a searchable index of source objects for a defined scope.
185
+
186
+ ## Prerequisites
187
+
188
+ - [ ] Source credentials configured
189
+ - [ ] Scope defined (sites, date range)
190
+ - [ ] Manifest prepared
191
+
192
+ ## Procedure
193
+
194
+ 1. Prepare manifest YAML
195
+ 2. Run index build
196
+ 3. Verify object count
197
+ 4. Query to validate coverage
198
+
199
+ ## Manifests
200
+
201
+ See [Manifest Templates](/content/reference/manifests/) for examples.
202
+
203
+ ## Verification
204
+
205
+ \`\`\`bash
206
+ # Check index count
207
+ # ← CUSTOMIZE: your verification command
208
+ \`\`\`
209
+
210
+ Expected: object count matches source scope.
211
+ `,
212
+ },
213
+ {
214
+ path: "content/pipeline/extract.md",
215
+ content: (_ctx: TemplateContext) => `---
216
+ title: Content Extraction
217
+ description: Derive routing fields from file content
218
+ ---
219
+
220
+ # Content Extraction
221
+
222
+ <!-- ← CUSTOMIZE: Replace with your extraction rules -->
223
+
224
+ ## Objective
225
+
226
+ Derive routing fields from file content to enable correct placement.
227
+
228
+ ## Configuration
229
+
230
+ <!-- Extraction rules: XPath, regex, JSON path -->
231
+
232
+ | Field | Source | Extraction Method |
233
+ |-------|--------|-------------------|
234
+ | date | File content | Regex / XPath |
235
+ | entity_id | Filename or content | Pattern match |
236
+ | category | Content analysis | Rules engine |
237
+
238
+ ## Procedure
239
+
240
+ 1. Prepare extraction config
241
+ 2. Run content probe
242
+ 3. Verify extracted fields
243
+ 4. Feed results to transfer stage
244
+
245
+ ## Verification
246
+
247
+ \`\`\`bash
248
+ # Spot-check extracted fields
249
+ # ← CUSTOMIZE: your verification command
250
+ \`\`\`
251
+ `,
252
+ },
253
+ {
254
+ path: "content/pipeline/transfer.md",
255
+ content: (_ctx: TemplateContext) => `---
256
+ title: Transfer / Reflow
257
+ description: Copy objects from source to destination with key rewriting
258
+ ---
259
+
260
+ # Transfer / Reflow
261
+
262
+ <!-- ← CUSTOMIZE: Replace with your transfer configuration -->
263
+
264
+ ## Objective
265
+
266
+ Copy objects from source to destination with key rewriting.
267
+
268
+ ## Configuration
269
+
270
+ - Source and destination profiles
271
+ - Rewrite templates
272
+ - Collision policy
273
+ - Checkpoint/resume settings
274
+
275
+ ## Procedure
276
+
277
+ 1. Prepare reflow command
278
+ 2. Dry-run first
279
+ 3. Execute with checkpoint
280
+ 4. Verify landing structure
281
+
282
+ ## Dry Run
283
+
284
+ \`\`\`bash
285
+ # ← CUSTOMIZE: your dry-run command
286
+ \`\`\`
287
+
288
+ **Verify**: Output shows expected file count and destination paths.
289
+
290
+ ## Execute
291
+
292
+ \`\`\`bash
293
+ # ← CUSTOMIZE: your transfer command with checkpoint
294
+ \`\`\`
295
+
296
+ ## Resume from Checkpoint
297
+
298
+ If interrupted:
299
+
300
+ \`\`\`bash
301
+ # ← CUSTOMIZE: your resume command
302
+ \`\`\`
303
+ `,
304
+ },
305
+ {
306
+ path: "content/pipeline/validate.md",
307
+ content: (_ctx: TemplateContext) => `---
308
+ title: Validation
309
+ description: Verify pipeline output is complete and correct
310
+ ---
311
+
312
+ # Validation
313
+
314
+ <!-- ← CUSTOMIZE: Replace with your validation checks -->
315
+
316
+ ## Objective
317
+
318
+ Verify pipeline output is complete and correct.
319
+
320
+ ## Checks
321
+
322
+ - [ ] Object count matches expected
323
+ - [ ] No duplicate files in destination
324
+ - [ ] Routing fields match destination paths
325
+ - [ ] No error records in output
326
+
327
+ ## Procedure
328
+
329
+ 1. Run validation report
330
+ 2. Compare source count vs destination count
331
+ 3. Check for duplicates
332
+ 4. Verify sample of routing decisions
333
+
334
+ ## Validation Report
335
+
336
+ \`\`\`bash
337
+ # ← CUSTOMIZE: your validation command
338
+ \`\`\`
339
+
340
+ ## Common Issues
341
+
342
+ | Issue | Cause | Resolution |
343
+ |-------|-------|------------|
344
+ | Count mismatch | Filtered items or errors | Check error log |
345
+ | Duplicates | Collision policy | Review collision settings |
346
+ | Wrong paths | Extraction rules | Verify field mappings |
347
+ `,
348
+ },
349
+ // Sources section
350
+ {
351
+ path: "content/sources/index.md",
352
+ content: (ctx: TemplateContext) => `---
353
+ title: Source Systems
354
+ description: Source data systems for ${ctx.branding.brandName}
355
+ ---
356
+
357
+ # Source Systems
358
+
359
+ <!-- ← CUSTOMIZE: Add your source systems -->
360
+
361
+ | Source | Type | Auth | Documentation |
362
+ |--------|------|------|---------------|
363
+ | *source-name* | S3 / GCS / file | Profile / keys | [Details](./source-name) |
364
+
365
+ ## Adding a New Source
366
+
367
+ 1. Create a file in \`content/sources/\` describing the system
368
+ 2. Document: connection details, auth profile, data format, schema
369
+ 3. Update this index table
370
+ `,
371
+ },
372
+ // Destinations section
373
+ {
374
+ path: "content/destinations/index.md",
375
+ content: (ctx: TemplateContext) => `---
376
+ title: Destinations
377
+ description: Target structures for ${ctx.branding.brandName}
378
+ ---
379
+
380
+ # Destinations
381
+
382
+ <!-- ← CUSTOMIZE: Add your destination systems -->
383
+
384
+ | Destination | Type | Structure | Documentation |
385
+ |-------------|------|-----------|---------------|
386
+ | *dest-name* | S3 / GCS / file | date-first / entity-first | [Details](./dest-name) |
387
+
388
+ ## Path Structure
389
+
390
+ <!-- ← CUSTOMIZE: Document your target path layout -->
391
+
392
+ \`\`\`
393
+ destination-root/
394
+ ├── {year}/
395
+ │ ├── {month}/
396
+ │ │ ├── {entity-id}/
397
+ │ │ │ └── {filename}
398
+ \`\`\`
399
+
400
+ ## Adding a New Destination
401
+
402
+ 1. Create a file in \`content/destinations/\` describing the target
403
+ 2. Document: path structure, naming conventions, access controls
404
+ 3. Update this index table
405
+ `,
406
+ },
407
+ // Operations section
408
+ {
409
+ path: "content/operations/run-pipeline.md",
410
+ content: (_ctx: TemplateContext) => `---
411
+ title: Run a Pipeline
412
+ description: Full pipeline execution procedure
413
+ ---
414
+
415
+ # Run a Pipeline
416
+
417
+ <!-- ← CUSTOMIZE: Replace with your pipeline commands -->
418
+
419
+ ## Prerequisites
420
+
421
+ - [ ] Source and destination credentials valid
422
+ - [ ] Index up to date for target scope
423
+ - [ ] Extraction config tested
424
+ - [ ] Dry-run completed
425
+
426
+ ## Full Pipeline Execution
427
+
428
+ ### Step 1: Index Build
429
+
430
+ \`\`\`bash
431
+ # ← CUSTOMIZE: your index build command
432
+ \`\`\`
433
+
434
+ **Verify**: Index count matches expected scope.
435
+
436
+ ### Step 2: Content Probe
437
+
438
+ \`\`\`bash
439
+ # ← CUSTOMIZE: your extraction command
440
+ \`\`\`
441
+
442
+ **Verify**: Extracted fields look correct for sample objects.
443
+
444
+ ### Step 3: Transfer Reflow
445
+
446
+ \`\`\`bash
447
+ # ← CUSTOMIZE: your transfer command
448
+ \`\`\`
449
+
450
+ **Verify**: Destination file count and structure.
451
+
452
+ ### Step 4: Validate
453
+
454
+ \`\`\`bash
455
+ # ← CUSTOMIZE: your validation command
456
+ \`\`\`
457
+
458
+ **Verify**: Validation report shows no errors.
459
+
460
+ ## Checkpoint / Resume
461
+
462
+ If interrupted, resume from checkpoint:
463
+
464
+ \`\`\`bash
465
+ # ← CUSTOMIZE: your resume command
466
+ \`\`\`
467
+
468
+ ## Post-Run
469
+
470
+ - [ ] Review validation report
471
+ - [ ] Update pipeline status table in [Home](/)
472
+ - [ ] Log run in [Schedules](/content/operations/schedules)
473
+ `,
474
+ },
475
+ {
476
+ path: "content/operations/schedules.md",
477
+ content: (_ctx: TemplateContext) => `---
478
+ title: Schedules
479
+ description: Pipeline execution schedules
480
+ ---
481
+
482
+ # Schedules
483
+
484
+ <!-- ← CUSTOMIZE: Add your pipeline schedules -->
485
+
486
+ | Pipeline | Frequency | Scope | Notes |
487
+ |----------|-----------|-------|-------|
488
+ | *pipeline-name* | Daily / Weekly / Ad hoc | *scope* | *notes* |
489
+
490
+ ## Schedule Management
491
+
492
+ - Automated pipelines run via cron/scheduler
493
+ - Ad hoc runs documented here after execution
494
+ - See [Run a Pipeline](/content/operations/run-pipeline) for execution steps
495
+ `,
496
+ },
497
+ // Troubleshooting section
498
+ {
499
+ path: "content/troubleshooting/common-issues.md",
500
+ content: (_ctx: TemplateContext) => `---
501
+ title: Common Issues
502
+ description: Troubleshooting guide for pipeline problems
503
+ ---
504
+
505
+ # Common Issues
506
+
507
+ <!-- ← CUSTOMIZE: Add your pipeline-specific issues -->
508
+
509
+ ## Authentication Expiry Mid-Pipeline
510
+
511
+ **Symptoms**:
512
+ - Pipeline fails partway through with auth errors
513
+ - "Access Denied" or "Token expired" in logs
514
+ - Partial results in destination
515
+
516
+ **Possible Causes**:
517
+ 1. Source credentials expired during long-running job
518
+ 2. Token refresh not configured
519
+ 3. Session timeout shorter than pipeline duration
520
+
521
+ **Resolution**:
522
+
523
+ 1. Check credential expiry:
524
+ \`\`\`bash
525
+ # ← CUSTOMIZE: your credential check command
526
+ \`\`\`
527
+
528
+ 2. Refresh credentials and resume from checkpoint
529
+ 3. Consider longer-lived tokens for large scopes
530
+
531
+ **Prevention**: Configure token refresh or use service account credentials.
532
+
533
+ ---
534
+
535
+ ## Index Lock / Corruption
536
+
537
+ **Symptoms**:
538
+ - "Index locked" or "Database is locked" errors
539
+ - Index build hangs or returns stale data
540
+ - Concurrent pipeline runs fail
541
+
542
+ **Resolution**:
543
+
544
+ 1. Check for other running pipeline processes
545
+ 2. Release lock if safe:
546
+ \`\`\`bash
547
+ # ← CUSTOMIZE: your lock release command
548
+ \`\`\`
549
+ 3. If corrupted, rebuild index from scratch
550
+
551
+ ---
552
+
553
+ ## Duplicate Detection Conflicts
554
+
555
+ **Symptoms**:
556
+ - Validation reports unexpected duplicates
557
+ - Same file appears in multiple destination paths
558
+ - Object count exceeds expected
559
+
560
+ **Resolution**:
561
+
562
+ 1. Review extraction rules — are routing fields unique?
563
+ 2. Check collision policy (overwrite vs skip vs error)
564
+ 3. Inspect duplicate pairs to find root cause
565
+
566
+ ---
567
+
568
+ ## Destination Collision Handling
569
+
570
+ **Symptoms**:
571
+ - Files overwritten unexpectedly
572
+ - "Collision" warnings in transfer log
573
+ - Missing files in destination
574
+
575
+ **Resolution**:
576
+
577
+ 1. Review collision policy setting
578
+ 2. Check if source contains genuine duplicates
579
+ 3. Adjust rewrite template to produce unique keys
580
+ `,
581
+ },
582
+ // Reference section subdirectories
583
+ {
584
+ path: "content/reference/manifests/job-template.md",
585
+ content: (_ctx: TemplateContext) => `---
586
+ title: Job Manifest Template
587
+ description: Template for pipeline job manifests
588
+ ---
589
+
590
+ # Job Manifest Template
591
+
592
+ <!-- ← CUSTOMIZE: Replace with your manifest format -->
593
+
594
+ \`\`\`yaml
595
+ # Pipeline Job Manifest
596
+ job:
597
+ name: "descriptive-job-name"
598
+ pipeline: "pipeline-id"
599
+ scope:
600
+ # ← CUSTOMIZE: define your scope fields
601
+ sites: []
602
+ date_range:
603
+ start: "YYYY-MM-DD"
604
+ end: "YYYY-MM-DD"
605
+ source:
606
+ profile: "source-profile-name"
607
+ destination:
608
+ profile: "dest-profile-name"
609
+ options:
610
+ dry_run: false
611
+ checkpoint: true
612
+ collision: "skip" # skip | overwrite | error
613
+ \`\`\`
614
+
615
+ ## Fields
616
+
617
+ | Field | Required | Description |
618
+ |-------|----------|-------------|
619
+ | \`job.name\` | Yes | Human-readable job identifier |
620
+ | \`job.pipeline\` | Yes | Pipeline definition to execute |
621
+ | \`job.scope\` | Yes | What data to process |
622
+ | \`job.source.profile\` | Yes | Source credential/connection profile |
623
+ | \`job.destination.profile\` | Yes | Destination credential/connection profile |
624
+ | \`job.options.dry_run\` | No | Preview without writing (default: false) |
625
+ | \`job.options.checkpoint\` | No | Enable resume capability (default: true) |
626
+ | \`job.options.collision\` | No | How to handle existing files (default: skip) |
627
+ `,
628
+ },
629
+ {
630
+ path: "content/reference/field-mappings/mapping-template.md",
631
+ content: (_ctx: TemplateContext) => `---
632
+ title: Field Mapping Template
633
+ description: Source field → destination field documentation
634
+ ---
635
+
636
+ # Field Mapping: [Source System]
637
+
638
+ <!-- ← CUSTOMIZE: Document your field mappings -->
639
+
640
+ ## Overview
641
+
642
+ | Source Field | Destination Field | Transform | Notes |
643
+ |-------------|-------------------|-----------|-------|
644
+ | \`source.field_a\` | \`dest.field_x\` | Direct copy | |
645
+ | \`source.field_b\` | \`dest.field_y\` | Date format | YYYY-MM-DD → YYYYMMDD |
646
+ | \`source.field_c\` | \`dest.field_z\` | Lookup table | See mapping table below |
647
+
648
+ ## Transform Rules
649
+
650
+ ### Date Formatting
651
+
652
+ Input: \`2024-03-15\` → Output: \`20240315\`
653
+
654
+ ### Lookup Tables
655
+
656
+ | Source Value | Destination Value |
657
+ |-------------|-------------------|
658
+ | \`type_a\` | \`A\` |
659
+ | \`type_b\` | \`B\` |
660
+ `,
661
+ },
662
+ {
663
+ path: "content/reference/metrics/pipeline-kpis.md",
664
+ content: (_ctx: TemplateContext) => `---
665
+ title: Pipeline KPIs
666
+ description: Key performance indicators for pipeline operations
667
+ ---
668
+
669
+ # Pipeline KPIs
670
+
671
+ <!-- ← CUSTOMIZE: Add your metrics and dashboard links -->
672
+
673
+ ## Key Metrics
674
+
675
+ | Metric | Target | Dashboard |
676
+ |--------|--------|-----------|
677
+ | Pipeline success rate | > 99% | [Link] |
678
+ | Average run duration | < X hours | [Link] |
679
+ | Object throughput | > X/hour | [Link] |
680
+ | Error rate | < 0.1% | [Link] |
681
+
682
+ ## SLAs
683
+
684
+ | Pipeline | Frequency | Completion Window | Escalation |
685
+ |----------|-----------|-------------------|------------|
686
+ | *pipeline-name* | Daily | 4 hours | Page on-call |
687
+
688
+ ## Alerting
689
+
690
+ | Alert | Warning | Critical | Action |
691
+ |-------|---------|----------|--------|
692
+ | Run duration | > 2x average | > 4x average | Investigate |
693
+ | Error count | > 10 | > 100 | Check logs |
694
+ | Stale index | > 24h | > 48h | Rebuild |
695
+ `,
696
+ },
697
+ {
698
+ path: "content/reference/checklists/pre-run.md",
699
+ content: (_ctx: TemplateContext) => `---
700
+ title: Pre-Run Checklist
701
+ description: Verification checklist before running a pipeline
702
+ ---
703
+
704
+ # Pre-Run Checklist
705
+
706
+ <!-- ← CUSTOMIZE: Adapt to your pipeline process -->
707
+
708
+ ## Credentials
709
+
710
+ - [ ] Source credentials valid and not expiring soon
711
+ - [ ] Destination credentials valid
712
+ - [ ] Service account permissions verified
713
+
714
+ ## Scope
715
+
716
+ - [ ] Scope definition reviewed (sites, date range)
717
+ - [ ] Expected object count estimated
718
+ - [ ] No overlapping concurrent runs
719
+
720
+ ## Configuration
721
+
722
+ - [ ] Manifest YAML validated
723
+ - [ ] Extraction rules tested on sample
724
+ - [ ] Destination path template verified
725
+ - [ ] Collision policy appropriate for this run
726
+
727
+ ## Environment
728
+
729
+ - [ ] Sufficient disk space for index
730
+ - [ ] Network connectivity to source and destination
731
+ - [ ] Monitoring and alerting configured
732
+
733
+ ## Go/No-Go
734
+
735
+ | Criteria | Status |
736
+ |----------|--------|
737
+ | Credentials valid | ⬜ |
738
+ | Scope defined | ⬜ |
739
+ | Config tested | ⬜ |
740
+ | Dry-run passed | ⬜ |
741
+
742
+ **Decision**: ⬜ GO / ⬜ NO-GO
743
+ `,
744
+ },
745
+ {
746
+ path: "content/reference/contacts/directory.md",
747
+ content: (ctx: TemplateContext) => `---
748
+ title: Contact Directory
749
+ description: Team and vendor contacts for ${ctx.branding.brandName}
750
+ ---
751
+
752
+ # Contact Directory
753
+
754
+ <!-- ← CUSTOMIZE: Add your team and vendor contacts -->
755
+
756
+ ## Pipeline Team
757
+
758
+ | Role | Contact | Responsibility |
759
+ |------|---------|----------------|
760
+ | Pipeline Lead | [Name/Email] | Architecture, operations |
761
+ | Data Engineer | [Name/Email] | Source/destination config |
762
+ | On-Call | [PagerDuty/Slack] | Incident response |
763
+
764
+ ## Vendor Contacts
765
+
766
+ | Vendor | Type | Contact |
767
+ |--------|------|---------|
768
+ | Cloud Provider | Support | [Portal/Email] |
769
+ | Data Source | Support | [Portal/Email] |
770
+
771
+ ## Escalation
772
+
773
+ For pipeline failures during business hours, contact the Pipeline Lead.
774
+ For off-hours incidents, page on-call via PagerDuty.
775
+ `,
776
+ },
777
+ // CUSTOMIZING.md
778
+ {
779
+ path: "CUSTOMIZING.md",
780
+ content: (ctx: TemplateContext) => `---
781
+ template: pipeline
782
+ template_version: 1
783
+ created: ${new Date().toISOString().split("T")[0]}
784
+ ---
785
+
786
+ # Customizing ${ctx.branding.siteName}
787
+
788
+ This guide helps you (and AI assistants) understand how to customize this pipeline operations site.
789
+
790
+ ## Site Structure
791
+
792
+ \`\`\`
793
+ ${ctx.name}/
794
+ ├── site.yaml # Site configuration (sections, branding)
795
+ ├── theme.yaml # Theme customization (create if needed)
796
+ ├── index.md # Home page with pipeline status
797
+ ├── CUSTOMIZING.md # This file
798
+ ├── content/
799
+ │ ├── pipeline/ # Pipeline stage documentation
800
+ │ ├── sources/ # Source data systems
801
+ │ ├── destinations/ # Target structures
802
+ │ ├── operations/ # Run procedures, schedules
803
+ │ ├── troubleshooting/ # Problem → solution guides
804
+ │ └── reference/ # Supporting materials
805
+ │ ├── manifests/ # Job manifest templates
806
+ │ ├── field-mappings/ # Source → destination mappings
807
+ │ ├── metrics/ # KPIs, SLAs, dashboards
808
+ │ ├── checklists/ # Pre-run, post-run checklists
809
+ │ └── contacts/ # Team contacts, escalation
810
+ └── assets/
811
+ └── brand/ # Logo, favicon
812
+ \`\`\`
813
+
814
+ ## Configuration Files
815
+
816
+ ### site.yaml - Site Configuration
817
+
818
+ \`\`\`yaml
819
+ title: "${ctx.branding.siteName}"
820
+
821
+ brand:
822
+ name: "${ctx.branding.brandName}" # Shown in header
823
+ url: "/" # Logo link destination
824
+
825
+ sections:
826
+ - name: "Pipeline"
827
+ path: "content/pipeline"
828
+ # Add or modify sections here
829
+ \`\`\`
830
+
831
+ ### theme.yaml - Visual Customization (optional)
832
+
833
+ Create \`theme.yaml\` to customize colors and typography:
834
+
835
+ \`\`\`yaml
836
+ colors:
837
+ primary: "#2563eb"
838
+ background: "#ffffff"
839
+ text: "#1f2937"
840
+
841
+ footer:
842
+ text: "© ${ctx.year} ${ctx.branding.brandName}"
843
+ \`\`\`
844
+
845
+ ## Adding Content
846
+
847
+ ### New Pipeline Stage
848
+
849
+ 1. Create file in \`content/pipeline/\`:
850
+ \`\`\`
851
+ content/pipeline/my-stage.md
852
+ \`\`\`
853
+
854
+ 2. Use the stage format:
855
+ \`\`\`markdown
856
+ ---
857
+ title: Stage Name
858
+ ---
859
+
860
+ # Stage Name
861
+
862
+ ## Objective
863
+ What this stage accomplishes.
864
+
865
+ ## Prerequisites
866
+ - [ ] Required items
867
+
868
+ ## Procedure
869
+ 1. Step with command
870
+ 2. Verification step
871
+
872
+ ## Verification
873
+ How to confirm the stage completed correctly.
874
+ \`\`\`
875
+
876
+ 3. Update the overview in \`content/pipeline/overview.md\`
877
+
878
+ ### New Source System
879
+
880
+ 1. Create file in \`content/sources/\`:
881
+ \`\`\`
882
+ content/sources/my-source.md
883
+ \`\`\`
884
+
885
+ 2. Document: connection details, auth profile, data format, schema
886
+ 3. Update the index table in \`content/sources/index.md\`
887
+
888
+ ### New Destination
889
+
890
+ 1. Create file in \`content/destinations/\`:
891
+ \`\`\`
892
+ content/destinations/my-destination.md
893
+ \`\`\`
894
+
895
+ 2. Document: path structure, naming conventions, access controls
896
+ 3. Update the index table in \`content/destinations/index.md\`
897
+
898
+ ### Creating Manifest Templates
899
+
900
+ 1. Create in \`content/reference/manifests/\`:
901
+ \`\`\`
902
+ content/reference/manifests/my-job.md
903
+ \`\`\`
904
+
905
+ 2. Include the full YAML template with field descriptions
906
+ 3. Document required vs optional fields
907
+
908
+ ### New Troubleshooting Guide
909
+
910
+ Follow the pattern: Symptoms → Causes → Resolution → Prevention
911
+
912
+ ### New Section
913
+
914
+ 1. Create folder: \`content/newsection/\`
915
+ 2. Add to \`site.yaml\`:
916
+ \`\`\`yaml
917
+ sections:
918
+ - name: "New Section"
919
+ path: "content/newsection"
920
+ \`\`\`
921
+ 3. Add at least one markdown file
922
+
923
+ ## Linking and References
924
+
925
+ ### Internal Links
926
+
927
+ \`\`\`markdown
928
+ See [Pipeline Overview](/content/pipeline/overview) for the full dataflow.
929
+ \`\`\`
930
+
931
+ ### External Links
932
+
933
+ \`\`\`markdown
934
+ See [Vendor Documentation](https://vendor.com/docs).
935
+ \`\`\`
936
+
937
+ ## Document Conventions
938
+
939
+ ### Pipeline Stages
940
+
941
+ - Start with Objective (what/why)
942
+ - List Prerequisites as checkboxes
943
+ - Number steps with verification
944
+ - Include expected outputs
945
+
946
+ ### Troubleshooting
947
+
948
+ - Lead with Symptoms (what user sees)
949
+ - List Possible Causes
950
+ - Provide step-by-step Resolution
951
+ - Include Prevention guidance
952
+
953
+ ### Checklists
954
+
955
+ - Use checkbox format: \`- [ ] Item\`
956
+ - Group by phase or category
957
+ - Include Go/No-Go decision point
958
+
959
+ ## Getting Help
960
+
961
+ - [Kitfly Documentation](https://github.com/3leaps/kitfly)
962
+ - [Markdown Guide](https://www.markdownguide.org/)
963
+ `,
964
+ },
965
+ ],
966
+ };