heimdall-agent 1.0.0 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (231) hide show
  1. package/README.md +57 -26
  2. package/README.pt-BR.md +429 -0
  3. package/agents/cloud-engineer.md +77 -0
  4. package/agents/code-sentinel.md +66 -35
  5. package/agents/database-architect.md +31 -25
  6. package/agents/frontend-designer.md +30 -28
  7. package/agents/mobile-designer.md +25 -24
  8. package/agents/modernizer.md +66 -0
  9. package/agents/orchestrator.md +94 -93
  10. package/agents/reviewer.md +100 -71
  11. package/agents/security-auditor.md +27 -27
  12. package/agents/tester.md +97 -90
  13. package/commands/grill-me.md +20 -4
  14. package/commands/orchestrate.md +16 -0
  15. package/commands/orchestrator.md +16 -0
  16. package/commands/plan.md +17 -1
  17. package/config/agents.json +21 -3
  18. package/config/mcp.json +72 -0
  19. package/config/models.json +26 -16
  20. package/config/profiles.json +2 -4
  21. package/config/skills.json +210 -100
  22. package/docs/SKILLS.md +341 -245
  23. package/docs/designs/old/2026-10-31 plan-lifecycle-backlog-and-clean-code-skill.md +87 -0
  24. package/docs/designs/old/2026-11-02 skill-upstream-audit-and-update.md +105 -0
  25. package/docs/designs/old/2026-11-05 communication-doctrine-bluf-ste.md +94 -0
  26. package/docs/designs/old/2026-11-06-cloud-engineer-and-resilience-synthesis.md +67 -0
  27. package/docs/designs/old/2026-11-06-herdr-native-agent-orchestration.md +45 -0
  28. package/docs/runs/skill-audit/add_progressive_disclosure.cjs +30 -0
  29. package/docs/runs/skill-audit/check_heimdall_skills.ps1 +9 -0
  30. package/docs/runs/skill-audit/compare_classics.ps1 +26 -0
  31. package/docs/runs/skill-audit/compare_taste.ps1 +12 -0
  32. package/docs/runs/skill-audit/customization-report.json +380 -0
  33. package/docs/runs/skill-audit/diff_pending_skills.cjs +94 -0
  34. package/docs/runs/skill-audit/fetch_and_apply_skill.cjs +58 -0
  35. package/docs/runs/skill-audit/fetch_and_apply_taste_v2.cjs +65 -0
  36. package/docs/runs/skill-audit/inspect_pending_skills.cjs +71 -0
  37. package/docs/runs/skill-audit/inventory.json +308 -0
  38. package/docs/runs/skill-audit/list_local_taste.ps1 +1 -0
  39. package/docs/runs/skill-audit/phase5-update-plan.json +595 -0
  40. package/docs/runs/skill-audit/phase5-update-plan.md +138 -0
  41. package/docs/runs/skill-audit/probe.ps1 +16 -0
  42. package/docs/runs/skill-audit/probe_classics_branches.ps1 +3 -0
  43. package/docs/runs/skill-audit/probe_commits.ps1 +15 -0
  44. package/docs/runs/skill-audit/probe_repos.ps1 +17 -0
  45. package/docs/runs/skill-audit/probe_sentry_sec.ps1 +3 -0
  46. package/docs/runs/skill-audit/probe_sentry_sweep.ps1 +5 -0
  47. package/docs/runs/skill-audit/probe_snyk.ps1 +3 -0
  48. package/docs/runs/skill-audit/probe_snyk_recipes.ps1 +5 -0
  49. package/docs/runs/skill-audit/probe_taste_branches.ps1 +8 -0
  50. package/docs/runs/skill-audit/probe_taste_v19.ps1 +3 -0
  51. package/docs/runs/skill-audit/probe_user_repos.ps1 +19 -0
  52. package/docs/runs/skill-audit/prune_taste_skills.cjs +22 -0
  53. package/docs/runs/skill-audit/run_phase1_2.cjs +142 -0
  54. package/docs/runs/skill-audit/run_phase6_verification.cjs +107 -0
  55. package/docs/runs/skill-audit/show_16_details.ps1 +25 -0
  56. package/docs/runs/skill-audit/show_20_skills.ps1 +28 -0
  57. package/docs/runs/skill-audit/show_classics_diff.cjs +37 -0
  58. package/docs/runs/skill-audit/sync_agent_markdowns.cjs +30 -0
  59. package/docs/runs/skill-audit/taste-skill-upstream.md +542 -0
  60. package/docs/runs/skill-audit/update-manifest.md +88 -0
  61. package/docs/runs/skill-audit/update_config_skills.cjs +52 -0
  62. package/docs/runs/skill-audit/update_registry_phase7.cjs +83 -0
  63. package/docs/runs/skill-audit/upstream-status.json +563 -0
  64. package/docs/runs/skill-audit/verification-results.json +252 -0
  65. package/doctrine/AGENTS.md +39 -77
  66. package/hooks/git/pre-commit +11 -0
  67. package/hooks/heimdall-hook.cjs +81 -2
  68. package/hooks/herdr-agent-state.ps1 +55 -0
  69. package/hooks/hooks.json +0 -11
  70. package/lib/catalog-sync.cjs +162 -0
  71. package/lib/hermes-context.cjs +57 -0
  72. package/lib/model-context.cjs +123 -0
  73. package/lib/path-discovery.cjs +94 -0
  74. package/package.json +7 -3
  75. package/playbooks/01-interactive-grilling-gate.md +14 -1
  76. package/playbooks/02-fast-code-deep-verification.md +3 -2
  77. package/playbooks/03-adversarial-code-review.md +3 -3
  78. package/playbooks/05-canonical-plan-template.md +32 -18
  79. package/playbooks/06-model-registry-matrix.md +24 -13
  80. package/playbooks/07-legacy-code-modernization.md +74 -0
  81. package/playbooks/08-backlog-and-change-request-flow.md +122 -0
  82. package/setup.mjs +592 -172
  83. package/skills/antislop/SKILL.md +76 -56
  84. package/skills/antislop/VERSION +1 -0
  85. package/skills/antislop-copywriting/SKILL.md +372 -372
  86. package/skills/antislop-human/LICENSE-web-quality-skills +21 -0
  87. package/skills/antislop-human/SKILL.md +204 -147
  88. package/skills/antislop-human/contrast-check.py +168 -165
  89. package/skills/antislop-human/contrast-mcp.py +6 -3
  90. package/skills/antislop-human/references/A11Y-PATTERNS.md +249 -0
  91. package/skills/antislop-human/references/WCAG.md +127 -0
  92. package/skills/antislop-ui/LICENSE-web-quality-skills +21 -0
  93. package/skills/antislop-ui/SKILL.md +344 -304
  94. package/skills/antislop-ui/references/CORE-WEB-VITALS.md +181 -0
  95. package/skills/antislop-ui/references/WEB-BASELINE.md +28 -0
  96. package/skills/brainstorming/SKILL.md +274 -265
  97. package/skills/brainstorming/builder-check-prompt.md +23 -0
  98. package/skills/brainstorming/scripts/frame-template.html +213 -213
  99. package/skills/brainstorming/scripts/server.cjs +723 -723
  100. package/skills/brandkit/SKILL.md +798 -798
  101. package/skills/brutalist-skill/SKILL.md +93 -92
  102. package/skills/caveman/README.md +68 -0
  103. package/skills/caveman/SKILL.md +73 -38
  104. package/skills/claude-collaborator/SKILL.md +24 -44
  105. package/skills/clean-code/SKILL.md +152 -0
  106. package/skills/cloudflare-security-audit/AI-AND-LLM.md +83 -83
  107. package/skills/cloudflare-security-audit/ATTACK-CLASSES.md +130 -130
  108. package/skills/cloudflare-security-audit/CLIENT-SIDE.md +83 -83
  109. package/skills/cloudflare-security-audit/CLOUD-AND-DEPLOYMENT.md +86 -86
  110. package/skills/cloudflare-security-audit/DATA-ISOLATION-AND-LIFECYCLE.md +84 -84
  111. package/skills/cloudflare-security-audit/DESKTOP-MOBILE-AND-LOCAL-IPC.md +89 -89
  112. package/skills/cloudflare-security-audit/HUNTING.md +251 -251
  113. package/skills/cloudflare-security-audit/MEMORY-SAFETY-AND-BINARY.md +101 -101
  114. package/skills/cloudflare-security-audit/PROTOCOLS-RPC-AND-MESSAGING.md +81 -81
  115. package/skills/cloudflare-security-audit/RECONNAISSANCE.md +156 -156
  116. package/skills/cloudflare-security-audit/RESOURCE-EXHAUSTION-AND-AVAILABILITY.md +78 -78
  117. package/skills/cloudflare-security-audit/SKILL.md +192 -192
  118. package/skills/cloudflare-security-audit/SUPPLY-CHAIN-AND-RELEASE.md +73 -73
  119. package/skills/cloudflare-security-audit/VALIDATION-AND-REPORTING.md +186 -186
  120. package/skills/cloudflare-security-audit/WEB-PROTOCOL-AND-AUTH.md +105 -105
  121. package/skills/cloudflare-security-audit/report-schema.json +461 -461
  122. package/skills/cloudflare-security-audit/validate-coverage-ledger.cjs +872 -872
  123. package/skills/cloudflare-security-audit/validate-coverage-ledger.test.cjs +740 -740
  124. package/skills/cloudflare-security-audit/validate-findings.cjs +773 -773
  125. package/skills/cloudflare-security-audit/validate-findings.test.cjs +652 -652
  126. package/skills/codex-collaborator/SKILL.md +24 -45
  127. package/skills/communication-doctrine/SKILL.md +89 -0
  128. package/skills/diagnosing-superpowers/SKILL.md +1 -1
  129. package/skills/diagnosing-superpowers/references/github-issues.md +3 -3
  130. package/skills/diagnosing-superpowers/templates/issue.md +23 -4
  131. package/skills/executing-plans/SKILL.md +0 -52
  132. package/skills/executing-plans/scripts/task-done +52 -52
  133. package/skills/executing-plans/scripts/task-start +28 -28
  134. package/skills/gemini-collaborator/SKILL.md +24 -56
  135. package/skills/herdr/SKILL.md +243 -0
  136. package/skills/image-to-code-skill/SKILL.md +1228 -1228
  137. package/skills/imagegen-frontend-mobile/SKILL.md +1465 -1465
  138. package/skills/imagegen-frontend-web/SKILL.md +987 -987
  139. package/skills/karpathy-guidelines/SKILL.md +67 -67
  140. package/skills/mcp-engineering/SKILL.md +74 -0
  141. package/skills/minimalist-skill/SKILL.md +87 -85
  142. package/skills/output-skill/SKILL.md +49 -49
  143. package/skills/ponytail/SKILL.md +36 -52
  144. package/skills/postgres-best-practices/SKILL.md +30 -52
  145. package/skills/postgres-best-practices/references/backup-restore.md +372 -0
  146. package/skills/postgres-best-practices/references/bulk-loading.md +315 -0
  147. package/skills/postgres-best-practices/references/connection-pooling.md +310 -0
  148. package/skills/postgres-best-practices/references/hot-standby.md +221 -0
  149. package/skills/postgres-best-practices/references/indexing.md +318 -0
  150. package/skills/postgres-best-practices/references/logical-replication.md +610 -0
  151. package/skills/postgres-best-practices/references/major-version-upgrades.md +334 -0
  152. package/skills/postgres-best-practices/references/performance-diagnostics.md +553 -0
  153. package/skills/postgres-best-practices/references/query-optimization.md +294 -0
  154. package/skills/postgres-best-practices/references/query-patterns.md +566 -0
  155. package/skills/postgres-best-practices/references/schema-design.md +362 -0
  156. package/skills/postgres-best-practices/references/security-roles.md +489 -0
  157. package/skills/postgres-best-practices/references/transaction-isolation.md +344 -0
  158. package/skills/progressive-disclosure/SKILL.md +68 -0
  159. package/skills/pstack-discipline/SKILL.md +40 -0
  160. package/skills/receiving-code-review/SKILL.md +2 -23
  161. package/skills/requesting-code-review/SKILL.md +7 -36
  162. package/skills/soft-skill/SKILL.md +99 -98
  163. package/skills/stitch-skill/DESIGN.md +121 -121
  164. package/skills/stitch-skill/SKILL.md +185 -184
  165. package/skills/strix-security/SKILL.md +54 -53
  166. package/skills/subagent-driven-development/SKILL.md +28 -260
  167. package/skills/subagent-driven-development/references/batching.md +11 -0
  168. package/skills/subagent-driven-development/references/cannot-verify.md +10 -0
  169. package/skills/subagent-driven-development/references/final-review.md +47 -0
  170. package/skills/subagent-driven-development/references/fix-loop.md +77 -0
  171. package/skills/subagent-driven-development/references/handling-reports.md +15 -0
  172. package/skills/subagent-driven-development/references/preflight.md +26 -0
  173. package/skills/subagent-driven-development/references/resuming.md +12 -0
  174. package/skills/subagent-driven-development/scripts/review-package +53 -53
  175. package/skills/subagent-driven-development/scripts/sdd-workspace +87 -82
  176. package/skills/subagent-driven-development/scripts/task-brief +43 -43
  177. package/skills/systematic-debugging/SKILL.md +283 -306
  178. package/skills/taste-skill/SKILL.md +542 -1206
  179. package/skills/taste-skill/references/app-screens.md +1199 -0
  180. package/skills/taste-skill/references/colour.md +141 -0
  181. package/skills/taste-skill/references/copy.md +177 -0
  182. package/skills/taste-skill/references/critique.md +182 -0
  183. package/skills/taste-skill/references/imagery.md +791 -0
  184. package/skills/taste-skill/references/layout.md +292 -0
  185. package/skills/taste-skill/references/motion.md +980 -0
  186. package/skills/taste-skill/references/research.md +356 -0
  187. package/skills/taste-skill/references/scenes.md +174 -0
  188. package/skills/taste-skill/references/tells.md +115 -0
  189. package/skills/taste-skill/references/type.md +170 -0
  190. package/skills/test-driven-development/SKILL.md +330 -337
  191. package/skills/thermo/SKILL.md +67 -0
  192. package/skills/threat-modeling/SKILL.md +55 -0
  193. package/skills/trailofbits-audit-context/SKILL.md +72 -72
  194. package/skills/trailofbits-insecure-defaults/commands/audit.md +35 -0
  195. package/skills/trailofbits-insecure-defaults/references/debug-features.json +13 -13
  196. package/skills/trailofbits-insecure-defaults/references/debug-features.md +75 -75
  197. package/skills/trailofbits-insecure-defaults/references/default-credentials.json +12 -12
  198. package/skills/trailofbits-insecure-defaults/references/default-credentials.md +76 -76
  199. package/skills/trailofbits-insecure-defaults/references/fail-open-security.json +11 -11
  200. package/skills/trailofbits-insecure-defaults/references/fail-open-security.md +70 -70
  201. package/skills/trailofbits-insecure-defaults/references/fallback-secrets.json +11 -11
  202. package/skills/trailofbits-insecure-defaults/references/fallback-secrets.md +70 -70
  203. package/skills/trailofbits-insecure-defaults/references/permissive-access.json +13 -13
  204. package/skills/trailofbits-insecure-defaults/references/permissive-access.md +62 -62
  205. package/skills/trailofbits-insecure-defaults/references/weak-crypto.json +13 -13
  206. package/skills/trailofbits-insecure-defaults/references/weak-crypto.md +71 -71
  207. package/skills/trailofbits-triage-brocards/SKILL.md +208 -208
  208. package/skills/verification-before-completion/SKILL.md +0 -14
  209. package/skills/writing-plans/SKILL.md +11 -15
  210. package/skills/writing-skills/SKILL.md +21 -303
  211. package/skills/writing-skills/references/CLAUDE_MD_TESTING.md +189 -0
  212. package/skills/writing-skills/references/anthropic-best-practices.md +1150 -0
  213. package/skills/writing-skills/references/bulletproofing.md +80 -0
  214. package/skills/writing-skills/references/discovery.md +201 -0
  215. package/skills/writing-skills/references/flowcharts.md +37 -0
  216. package/skills/writing-skills/references/graphviz-conventions.dot +172 -0
  217. package/skills/writing-skills/references/persuasion-principles.md +187 -0
  218. package/skills/writing-skills/references/testing-skills-with-subagents.md +384 -0
  219. package/tools/{claude-bridge.ps1 → archive/claude-bridge.ps1} +10 -7
  220. package/tools/{codex-bridge.ps1 → archive/codex-bridge.ps1} +1 -1
  221. package/tools/{gemini-bridge.ps1 → archive/gemini-bridge.ps1} +12 -19
  222. package/skills/gpt-tasteskill/SKILL.md +0 -74
  223. package/skills/taste-skill-v1/SKILL.md +0 -226
  224. /package/tools/{claude-bridge.cmd → archive/claude-bridge.cmd} +0 -0
  225. /package/tools/{claude-bridge.sh → archive/claude-bridge.sh} +0 -0
  226. /package/tools/{codex-bridge.cmd → archive/codex-bridge.cmd} +0 -0
  227. /package/tools/{codex-bridge.sh → archive/codex-bridge.sh} +0 -0
  228. /package/tools/{codexcli-bridge.cmd → archive/codexcli-bridge.cmd} +0 -0
  229. /package/tools/{codexcli-bridge.ps1 → archive/codexcli-bridge.ps1} +0 -0
  230. /package/tools/{gemini-bridge.cmd → archive/gemini-bridge.cmd} +0 -0
  231. /package/tools/{gemini-bridge.sh → archive/gemini-bridge.sh} +0 -0
@@ -0,0 +1,315 @@
1
+ # Bulk Data Loading Reference
2
+
3
+ ## Contents
4
+ - COPY vs INSERT performance
5
+ - COPY FROM patterns
6
+ - Optimizing large loads
7
+ - ETL staging patterns
8
+ - Bulk updates and deletes
9
+
10
+ ## COPY vs INSERT Performance
11
+
12
+ | Method | Rows/sec (typical) | Use case |
13
+ |--------|-------------------|----------|
14
+ | Single-row INSERT | ~1,000-5,000 | Application writes |
15
+ | Multi-row INSERT (VALUES) | ~10,000-50,000 | Batch inserts from code |
16
+ | `COPY FROM` | ~100,000-500,000+ | Bulk loading from files or streams |
17
+ | `COPY FROM` (binary) | ~200,000-1,000,000+ | Maximum throughput (binary format) |
18
+
19
+ `COPY` is **10-100x faster** than individual INSERTs because it bypasses per-row overhead (parsing, planning, WAL per statement).
20
+
21
+ ## COPY FROM Patterns
22
+
23
+ ### From a File
24
+
25
+ ```sql
26
+ -- CSV with header
27
+ COPY orders FROM '/path/to/orders.csv' WITH (FORMAT csv, HEADER true);
28
+
29
+ -- Tab-delimited
30
+ COPY orders FROM '/path/to/orders.tsv' WITH (FORMAT text);
31
+
32
+ -- Custom delimiter
33
+ COPY orders FROM '/path/to/orders.dat' WITH (DELIMITER '|');
34
+
35
+ -- With NULL handling
36
+ COPY orders FROM '/path/to/orders.csv'
37
+ WITH (FORMAT csv, HEADER true, NULL '');
38
+ ```
39
+
40
+ ### From STDIN (Piped Data)
41
+
42
+ ```bash
43
+ # Pipe from another command
44
+ cat orders.csv | psql -d mydb -c "\COPY orders FROM STDIN WITH (FORMAT csv, HEADER true)"
45
+
46
+ # Pipe from gzip
47
+ gunzip -c orders.csv.gz | psql -d mydb -c "\COPY orders FROM STDIN WITH (FORMAT csv, HEADER true)"
48
+ ```
49
+
50
+ ### From Application Code
51
+
52
+ Most drivers support COPY protocol for streaming data:
53
+
54
+ ```python
55
+ # Python (psycopg 3)
56
+ with conn.cursor() as cur:
57
+ with cur.copy("COPY orders (id, customer_id, total) FROM STDIN") as copy:
58
+ for row in data:
59
+ copy.write_row(row)
60
+ ```
61
+
62
+ ```javascript
63
+ // Node.js (pg-copy-streams)
64
+ const { from } = require('pg-copy-streams');
65
+ const stream = client.query(from('COPY orders FROM STDIN WITH (FORMAT csv)'));
66
+ fileStream.pipe(stream);
67
+ ```
68
+
69
+ ### COPY TO (Export)
70
+
71
+ ```sql
72
+ -- Export to CSV
73
+ COPY orders TO '/path/to/orders.csv' WITH (FORMAT csv, HEADER true);
74
+
75
+ -- Export query results
76
+ COPY (SELECT id, total FROM orders WHERE created_at > '2024-01-01')
77
+ TO '/path/to/recent.csv' WITH (FORMAT csv, HEADER true);
78
+ ```
79
+
80
+ ### Error Handling (PG17+)
81
+
82
+ ```sql
83
+ -- Skip invalid rows instead of failing the entire COPY
84
+ COPY orders FROM '/path/to/orders.csv'
85
+ WITH (FORMAT csv, HEADER true, ON_ERROR ignore);
86
+
87
+ -- PG18+: limit how many errors to tolerate
88
+ COPY orders FROM '/path/to/orders.csv'
89
+ WITH (FORMAT csv, HEADER true, ON_ERROR ignore, REJECT_LIMIT 100);
90
+ ```
91
+
92
+ ## Optimizing Large Loads
93
+
94
+ ### 1. Drop Indexes, Load, Recreate
95
+
96
+ Building indexes incrementally during COPY is much slower than building them once after:
97
+
98
+ ```sql
99
+ -- Before load: drop non-essential indexes
100
+ DROP INDEX idx_orders_customer_id;
101
+ DROP INDEX idx_orders_created_at;
102
+
103
+ -- Load data
104
+ COPY orders FROM '/path/to/orders.csv' WITH (FORMAT csv, HEADER true);
105
+
106
+ -- After load: recreate indexes (concurrently if table is in use)
107
+ CREATE INDEX idx_orders_customer_id ON orders(customer_id);
108
+ CREATE INDEX idx_orders_created_at ON orders(created_at);
109
+
110
+ -- Update statistics
111
+ ANALYZE orders;
112
+ ```
113
+
114
+ ### 2. Disable Triggers During Load
115
+
116
+ ```sql
117
+ -- Disable all triggers on the table
118
+ ALTER TABLE orders DISABLE TRIGGER ALL;
119
+
120
+ -- Load data
121
+ COPY orders FROM '/path/to/orders.csv' WITH (FORMAT csv, HEADER true);
122
+
123
+ -- Re-enable triggers
124
+ ALTER TABLE orders ENABLE TRIGGER ALL;
125
+ ```
126
+
127
+ `DISABLE TRIGGER ALL` requires superuser privileges when the table has foreign-key or other internally generated constraint triggers. A table owner can use `DISABLE TRIGGER USER` to disable only user-defined triggers. Disabling constraint triggers can admit invalid data, so validate constraints before re-enabling writes.
128
+
129
+ ### 3. Increase maintenance_work_mem
130
+
131
+ Larger `maintenance_work_mem` speeds up index creation after the load:
132
+
133
+ ```sql
134
+ SET maintenance_work_mem = '1GB'; -- for the duration of the load session
135
+ ```
136
+
137
+ ### 4. Disable Autovacuum During Load
138
+
139
+ For very large bulk loads, temporarily disable autovacuum to avoid competing I/O:
140
+
141
+ ```sql
142
+ ALTER TABLE orders SET (autovacuum_enabled = false);
143
+
144
+ -- Load data...
145
+
146
+ ALTER TABLE orders SET (autovacuum_enabled = true);
147
+ VACUUM ANALYZE orders;
148
+ ```
149
+
150
+ ### 5. Use Unlogged Tables for Staging
151
+
152
+ Unlogged tables skip WAL writes — 2-3x faster for writes but **data is lost on crash**:
153
+
154
+ ```sql
155
+ CREATE UNLOGGED TABLE staging_orders (LIKE orders INCLUDING ALL);
156
+
157
+ COPY staging_orders FROM '/path/to/orders.csv' WITH (FORMAT csv, HEADER true);
158
+
159
+ -- Transform and move to the real table
160
+ INSERT INTO orders SELECT * FROM staging_orders;
161
+
162
+ DROP TABLE staging_orders;
163
+ ```
164
+
165
+ ### 6. Batch Size for Programmatic Inserts
166
+
167
+ When COPY isn't available, use multi-row INSERT with batches of 100-1,000 rows:
168
+
169
+ ```sql
170
+ -- Single round-trip for 1,000 rows
171
+ INSERT INTO orders (customer_id, total, created_at)
172
+ VALUES
173
+ (1, 99.99, now()),
174
+ (2, 149.99, now()),
175
+ -- ... up to ~1,000 rows per statement
176
+ (1000, 79.99, now());
177
+ ```
178
+
179
+ Beyond ~1,000 rows per statement, parse overhead increases. Use COPY for larger batches.
180
+
181
+ ### 7. Parallel Loading into Partitioned Tables
182
+
183
+ Load data into individual partitions concurrently from separate sessions:
184
+
185
+ ```bash
186
+ # Session 1
187
+ psql -c "COPY events_2024_q1 FROM '/data/q1.csv' WITH (FORMAT csv)"
188
+
189
+ # Session 2 (concurrent)
190
+ psql -c "COPY events_2024_q2 FROM '/data/q2.csv' WITH (FORMAT csv)"
191
+ ```
192
+
193
+ ## ETL Staging Patterns
194
+
195
+ ### Staging Table with Upsert
196
+
197
+ ```sql
198
+ -- Create staging table (temporary or unlogged)
199
+ CREATE TEMP TABLE staging_customers (LIKE customers INCLUDING DEFAULTS);
200
+
201
+ -- Load raw data
202
+ COPY staging_customers FROM '/path/to/customers.csv' WITH (FORMAT csv, HEADER true);
203
+
204
+ -- Upsert into production table
205
+ INSERT INTO customers (id, name, email, updated_at)
206
+ SELECT id, name, email, now()
207
+ FROM staging_customers
208
+ ON CONFLICT (id)
209
+ DO UPDATE SET
210
+ name = EXCLUDED.name,
211
+ email = EXCLUDED.email,
212
+ updated_at = EXCLUDED.updated_at;
213
+ ```
214
+
215
+ ### Staging Table with MERGE (PG15+)
216
+
217
+ ```sql
218
+ MERGE INTO customers AS target
219
+ USING staging_customers AS source
220
+ ON target.id = source.id
221
+ WHEN MATCHED AND source.name IS DISTINCT FROM target.name THEN
222
+ UPDATE SET name = source.name, email = source.email, updated_at = now()
223
+ WHEN NOT MATCHED THEN
224
+ INSERT (id, name, email, updated_at)
225
+ VALUES (source.id, source.name, source.email, now());
226
+ ```
227
+
228
+ ### Swap Table Pattern
229
+
230
+ For full-refresh loads where you replace all data:
231
+
232
+ ```sql
233
+ -- Load into a new table
234
+ CREATE TABLE orders_new (LIKE orders INCLUDING ALL);
235
+ COPY orders_new FROM '/path/to/orders.csv' WITH (FORMAT csv, HEADER true);
236
+ ANALYZE orders_new;
237
+
238
+ -- Atomic swap (brief exclusive lock)
239
+ BEGIN;
240
+ ALTER TABLE orders RENAME TO orders_old;
241
+ ALTER TABLE orders_new RENAME TO orders;
242
+ DROP TABLE orders_old;
243
+ COMMIT;
244
+ ```
245
+
246
+ `LIKE ... INCLUDING ALL` does not copy foreign keys, triggers, rules, grants, row-level security policies, or publication membership. Recreate and verify those objects before swapping, or use a data-only refresh when the original table's identity and dependencies must remain unchanged.
247
+
248
+ ## Bulk Updates and Deletes
249
+
250
+ ### Chunked Deletes
251
+
252
+ Large DELETEs lock rows and generate WAL. This loop limits each statement to 10,000 rows:
253
+
254
+ ```sql
255
+ -- Delete in batches of 10,000
256
+ DO $$
257
+ DECLARE
258
+ rows_deleted int;
259
+ BEGIN
260
+ LOOP
261
+ DELETE FROM audit_log
262
+ WHERE id IN (
263
+ SELECT id FROM audit_log
264
+ WHERE created_at < now() - interval '1 year'
265
+ LIMIT 10000
266
+ );
267
+ GET DIAGNOSTICS rows_deleted = ROW_COUNT;
268
+ EXIT WHEN rows_deleted = 0;
269
+ -- Optional: brief pause to reduce WAL pressure
270
+ PERFORM pg_sleep(0.1);
271
+ END LOOP;
272
+ END $$;
273
+ ```
274
+
275
+ The entire `DO` block is still one transaction: locks, WAL, and dead rows accumulate until it finishes. For true transaction-level batching, execute one limited DELETE per client transaction and commit between batches.
276
+
277
+ ### Chunked Updates
278
+
279
+ The same statement-size pattern works for large updates:
280
+
281
+ ```sql
282
+ DO $$
283
+ DECLARE
284
+ rows_updated int;
285
+ BEGIN
286
+ LOOP
287
+ UPDATE orders
288
+ SET status = 'archived'
289
+ WHERE id IN (
290
+ SELECT id FROM orders
291
+ WHERE status = 'completed'
292
+ AND created_at < now() - interval '2 years'
293
+ LIMIT 10000
294
+ FOR UPDATE SKIP LOCKED
295
+ );
296
+ GET DIAGNOSTICS rows_updated = ROW_COUNT;
297
+ EXIT WHEN rows_updated = 0;
298
+ END LOOP;
299
+ END $$;
300
+ ```
301
+
302
+ As with the DELETE loop, the `DO` block commits only once. Drive batches from the client or a transaction-controlling procedure invoked outside an explicit transaction when each batch must commit independently.
303
+
304
+ ### Partition Drop Instead of Delete
305
+
306
+ If data is partitioned by time, dropping a partition is instant vs. a slow DELETE:
307
+
308
+ ```sql
309
+ -- Instant: drop old partition
310
+ ALTER TABLE events DETACH PARTITION events_2023_q1 CONCURRENTLY;
311
+ DROP TABLE events_2023_q1;
312
+
313
+ -- vs. slow: delete rows
314
+ -- DELETE FROM events WHERE occurred_at < '2024-01-01'; -- don't do this
315
+ ```
@@ -0,0 +1,310 @@
1
+ # Connection Pooling Reference
2
+
3
+ ## Contents
4
+ - Why connection pooling matters
5
+ - PgBouncer configuration
6
+ - Pool modes
7
+ - Prepared statement handling
8
+ - Monitoring and diagnostics
9
+ - Application-side pooling
10
+
11
+ ## Why Connection Pooling Matters
12
+
13
+ Each PostgreSQL connection spawns a dedicated backend process (~5-10 MB of memory). Without pooling:
14
+
15
+ - 500 application instances × 10 connections each = 5,000 backend processes
16
+ - Memory: 5,000 × 10 MB = ~50 GB just for connection overhead
17
+ - Context switching degrades performance above a few hundred active backends
18
+ - `max_connections` must be set high, wasting shared memory
19
+
20
+ A connection pooler sits between the application and PostgreSQL, multiplexing many client connections onto fewer server connections.
21
+
22
+ ### Connection Overhead
23
+
24
+ ```sql
25
+ -- Current connections vs. limit
26
+ SELECT
27
+ count(*) AS current,
28
+ (SELECT setting::int FROM pg_settings WHERE name = 'max_connections') AS max,
29
+ (SELECT setting::int FROM pg_settings WHERE name = 'superuser_reserved_connections') AS reserved
30
+ FROM pg_stat_activity;
31
+
32
+ -- Potential per-operation and per-session memory settings
33
+ SELECT
34
+ name,
35
+ current_setting(name) AS configured_value,
36
+ pg_size_bytes(current_setting(name)) AS configured_bytes
37
+ FROM pg_settings
38
+ WHERE name IN ('work_mem', 'temp_buffers')
39
+ ORDER BY name;
40
+ ```
41
+
42
+ These are not baseline allocations per backend. `work_mem` can be consumed by multiple query operations, while `temp_buffers` is allocated lazily when a session uses temporary tables.
43
+
44
+ ## PgBouncer Configuration
45
+
46
+ PgBouncer is the most widely used connection pooler for PostgreSQL.
47
+
48
+ ### Essential pgbouncer.ini Settings
49
+
50
+ ```ini
51
+ [databases]
52
+ mydb = host=127.0.0.1 port=5432 dbname=mydb
53
+
54
+ [pgbouncer]
55
+ listen_addr = 0.0.0.0
56
+ listen_port = 6432
57
+ auth_type = scram-sha-256
58
+ auth_file = /etc/pgbouncer/userlist.txt
59
+
60
+ # Pool sizing
61
+ pool_mode = transaction
62
+ default_pool_size = 20
63
+ min_pool_size = 5
64
+ max_client_conn = 1000
65
+ max_db_connections = 50
66
+
67
+ # Timeouts
68
+ server_idle_timeout = 300
69
+ client_idle_timeout = 0
70
+ query_timeout = 0
71
+ query_wait_timeout = 120
72
+ server_login_retry = 15
73
+
74
+ # Logging
75
+ log_connections = 0
76
+ log_disconnections = 0
77
+ stats_period = 60
78
+ ```
79
+
80
+ ### Key Parameters
81
+
82
+ | Parameter | Recommended | Purpose |
83
+ |-----------|------------|---------|
84
+ | `default_pool_size` | 20-50 | Server connections per user/database pair |
85
+ | `min_pool_size` | 5 | Minimum idle server connections to keep open |
86
+ | `max_client_conn` | 1000-10000 | Max client connections PgBouncer accepts |
87
+ | `max_db_connections` | 50-100 | Hard cap on server connections per database |
88
+ | `reserve_pool_size` | 5 | Extra connections for burst traffic |
89
+ | `reserve_pool_timeout` | 3 | Seconds before using reserve pool |
90
+ | `query_wait_timeout` | 120 | Max time a client waits for a server connection |
91
+
92
+ ### Sizing Rule of Thumb
93
+
94
+ **Server connections**: Set `default_pool_size` to roughly 2-4x the number of CPU cores on the database server. More connections don't help — they just increase lock contention and context switching.
95
+
96
+ **Client connections**: Set `max_client_conn` high enough for all application instances. PgBouncer handles thousands of idle client connections with minimal memory.
97
+
98
+ ```
99
+ App instances (500 clients) ──→ PgBouncer (max_client_conn=1000) ──→ PostgreSQL (pool_size=20)
100
+ ```
101
+
102
+ ## Pool Modes
103
+
104
+ ### Transaction Mode (Recommended for Most Applications)
105
+
106
+ ```ini
107
+ pool_mode = transaction
108
+ ```
109
+
110
+ Server connection is assigned when a transaction begins and returned when it commits/rolls back. Between transactions, the connection is available to other clients.
111
+
112
+ **Compatible with**: Standard SQL, parameterized queries, most ORMs.
113
+
114
+ **NOT compatible with** (session-level features):
115
+ - `SET` / `RESET` (use `SET LOCAL` inside a transaction instead)
116
+ - `LISTEN` / `NOTIFY`
117
+ - SQL-level `PREPARE` / `DEALLOCATE` (use protocol-level prepared statements)
118
+ - `DECLARE ... WITH HOLD` cursors
119
+ - Temporary tables with `ON COMMIT PRESERVE ROWS`
120
+ - Session-level advisory locks (`pg_advisory_lock` — use `pg_advisory_xact_lock` instead)
121
+ - `LOAD` statement
122
+
123
+ ### Session Mode
124
+
125
+ ```ini
126
+ pool_mode = session
127
+ ```
128
+
129
+ Server connection is held for the entire client session. Compatible with all PostgreSQL features but offers less multiplexing benefit.
130
+
131
+ **Use when**: Application relies on session-level features (temp tables, LISTEN/NOTIFY, SET parameters).
132
+
133
+ ### Statement Mode
134
+
135
+ ```ini
136
+ pool_mode = statement
137
+ ```
138
+
139
+ Server connection released after every statement. Maximum multiplexing but **incompatible with multi-statement transactions**.
140
+
141
+ **Use when**: Application only runs autocommit single statements (rare).
142
+
143
+ ### Choosing a Mode
144
+
145
+ | Application pattern | Recommended mode |
146
+ |--------------------|-----------------|
147
+ | Web apps, APIs, serverless | Transaction |
148
+ | Applications using LISTEN/NOTIFY | Session |
149
+ | Applications using temp tables across transactions | Session |
150
+ | Applications with SET session variables | Session (or refactor to `SET LOCAL`) |
151
+ | Simple autocommit queries | Statement |
152
+
153
+ ## Prepared Statement Handling
154
+
155
+ ### The Problem
156
+
157
+ SQL-level `PREPARE`/`EXECUTE` creates server-side prepared statements tied to a session. In transaction mode, the next transaction may use a different server connection where the prepared statement doesn't exist.
158
+
159
+ ### Solutions
160
+
161
+ **Protocol-level prepared statements**: Most modern drivers use the PostgreSQL wire protocol's `Parse`/`Bind`/`Execute` messages, which PgBouncer (1.21+) can handle:
162
+
163
+ ```ini
164
+ # PgBouncer 1.21+
165
+ max_prepared_statements = 100 # per server connection
166
+ ```
167
+
168
+ **Driver-level configuration** to avoid SQL-level PREPARE:
169
+
170
+ ```python
171
+ # Python psycopg: uses protocol-level by default — no change needed
172
+ ```
173
+
174
+ ```javascript
175
+ // Node.js pg: protocol-level by default
176
+ const pool = new Pool({ ...config });
177
+ // For explicit control:
178
+ // statement_timeout via SET LOCAL, not SET
179
+ ```
180
+
181
+ ```java
182
+ // JDBC: protocol-level by default with prepareThreshold
183
+ // ?prepareThreshold=5 (default: 5 uses before server-side prepare)
184
+ // ?prepareThreshold=0 (disable server-side prepare entirely)
185
+ ```
186
+
187
+ ```ruby
188
+ # Ruby pg: protocol-level by default
189
+ # ActiveRecord: no special config needed
190
+ ```
191
+
192
+ ## Monitoring and Diagnostics
193
+
194
+ ### PgBouncer Admin Console
195
+
196
+ Connect to PgBouncer's admin port:
197
+
198
+ ```bash
199
+ psql -h 127.0.0.1 -p 6432 -U pgbouncer pgbouncer
200
+ ```
201
+
202
+ ### Key Commands
203
+
204
+ ```text
205
+ -- Pool status (most useful)
206
+ SHOW POOLS;
207
+ -- Columns: database, user, cl_active, cl_waiting, sv_active, sv_idle, sv_used, pool_mode
208
+
209
+ -- Active client and server connections
210
+ SHOW CLIENTS;
211
+ SHOW SERVERS;
212
+
213
+ -- Aggregate statistics
214
+ SHOW STATS;
215
+ -- Columns: total_xact_count, total_query_count, avg_xact_time, avg_query_time
216
+
217
+ -- Configuration
218
+ SHOW CONFIG;
219
+
220
+ -- Memory usage
221
+ SHOW MEM;
222
+ ```
223
+
224
+ These commands use PgBouncer's admin protocol and fail if sent directly to PostgreSQL.
225
+
226
+ ### What to Watch
227
+
228
+ | Metric | Healthy | Problem |
229
+ |--------|---------|---------|
230
+ | `cl_waiting` | 0 | > 0 = clients waiting for server connections |
231
+ | `sv_active` | < pool_size | = pool_size = pool exhausted |
232
+ | `sv_idle` | > 0 | 0 = no spare connections |
233
+ | `avg_xact_time` | < 100ms | High = long transactions hogging connections |
234
+ | `avg_wait_time` | 0 | > 0 = pool too small or transactions too long |
235
+
236
+ ### Common Issues
237
+
238
+ **Clients waiting (`cl_waiting > 0`)**:
239
+ 1. Check for long-running transactions: `SELECT * FROM pg_stat_activity WHERE state = 'idle in transaction';`
240
+ 2. Increase `default_pool_size` (but diminishing returns beyond ~4x CPU cores)
241
+ 3. Check if `query_wait_timeout` is being hit (errors in application logs)
242
+
243
+ **Server connections not being returned**:
244
+ 1. `idle in transaction` sessions hold connections — set `idle_in_transaction_session_timeout` in PostgreSQL
245
+ 2. Application not committing/rolling back — add explicit transaction management
246
+
247
+ ## Application-Side Pooling
248
+
249
+ When PgBouncer isn't available, most drivers offer built-in connection pooling.
250
+
251
+ ### Recommended Pool Sizes
252
+
253
+ **Per application instance**: 5-20 connections. Start low, increase only if you see connection wait times.
254
+
255
+ **Total across all instances**: Should not exceed ~4x database CPU cores for active connections.
256
+
257
+ ### Common Driver Configuration
258
+
259
+ ```python
260
+ # Python (psycopg pool)
261
+ from psycopg_pool import ConnectionPool
262
+
263
+ pool = ConnectionPool(
264
+ conninfo="host=db port=5432 dbname=mydb",
265
+ min_size=5,
266
+ max_size=20,
267
+ max_idle=300, # close idle connections after 5 min
268
+ )
269
+
270
+ with pool.connection() as conn:
271
+ conn.execute("SELECT ...")
272
+ ```
273
+
274
+ ```javascript
275
+ // Node.js (pg)
276
+ const { Pool } = require('pg');
277
+ const pool = new Pool({
278
+ host: 'db',
279
+ database: 'mydb',
280
+ max: 20, // max connections in pool
281
+ idleTimeoutMillis: 30000,
282
+ connectionTimeoutMillis: 5000,
283
+ });
284
+
285
+ const result = await pool.query('SELECT ...');
286
+ ```
287
+
288
+ ```java
289
+ // Java (HikariCP)
290
+ HikariConfig config = new HikariConfig();
291
+ config.setJdbcUrl("jdbc:postgresql://db:5432/mydb");
292
+ config.setMaximumPoolSize(20);
293
+ config.setMinimumIdle(5);
294
+ config.setIdleTimeout(300000);
295
+ config.setConnectionTimeout(5000);
296
+
297
+ HikariDataSource ds = new HikariDataSource(config);
298
+ ```
299
+
300
+ ### Application Pool + PgBouncer
301
+
302
+ You can stack both. Each application instance pools locally (5-10 connections), and PgBouncer pools across all instances:
303
+
304
+ ```
305
+ App1 (pool: 10) ──┐
306
+ App2 (pool: 10) ──┼──→ PgBouncer (pool: 30) ──→ PostgreSQL
307
+ App3 (pool: 10) ──┘
308
+ ```
309
+
310
+ Set application pool sizes low when using PgBouncer — the point is to reduce total connections, not add them up.