heimdall-agent 1.0.0 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +57 -26
- package/README.pt-BR.md +429 -0
- package/agents/cloud-engineer.md +77 -0
- package/agents/code-sentinel.md +66 -35
- package/agents/database-architect.md +31 -25
- package/agents/frontend-designer.md +30 -28
- package/agents/mobile-designer.md +25 -24
- package/agents/modernizer.md +66 -0
- package/agents/orchestrator.md +94 -93
- package/agents/reviewer.md +100 -71
- package/agents/security-auditor.md +27 -27
- package/agents/tester.md +97 -90
- package/commands/grill-me.md +20 -4
- package/commands/orchestrate.md +16 -0
- package/commands/orchestrator.md +16 -0
- package/commands/plan.md +17 -1
- package/config/agents.json +21 -3
- package/config/mcp.json +72 -0
- package/config/models.json +26 -16
- package/config/profiles.json +2 -4
- package/config/skills.json +210 -100
- package/docs/SKILLS.md +341 -245
- package/docs/designs/old/2026-10-31 plan-lifecycle-backlog-and-clean-code-skill.md +87 -0
- package/docs/designs/old/2026-11-02 skill-upstream-audit-and-update.md +105 -0
- package/docs/designs/old/2026-11-05 communication-doctrine-bluf-ste.md +94 -0
- package/docs/designs/old/2026-11-06-cloud-engineer-and-resilience-synthesis.md +67 -0
- package/docs/designs/old/2026-11-06-herdr-native-agent-orchestration.md +45 -0
- package/docs/runs/skill-audit/add_progressive_disclosure.cjs +30 -0
- package/docs/runs/skill-audit/check_heimdall_skills.ps1 +9 -0
- package/docs/runs/skill-audit/compare_classics.ps1 +26 -0
- package/docs/runs/skill-audit/compare_taste.ps1 +12 -0
- package/docs/runs/skill-audit/customization-report.json +380 -0
- package/docs/runs/skill-audit/diff_pending_skills.cjs +94 -0
- package/docs/runs/skill-audit/fetch_and_apply_skill.cjs +58 -0
- package/docs/runs/skill-audit/fetch_and_apply_taste_v2.cjs +65 -0
- package/docs/runs/skill-audit/inspect_pending_skills.cjs +71 -0
- package/docs/runs/skill-audit/inventory.json +308 -0
- package/docs/runs/skill-audit/list_local_taste.ps1 +1 -0
- package/docs/runs/skill-audit/phase5-update-plan.json +595 -0
- package/docs/runs/skill-audit/phase5-update-plan.md +138 -0
- package/docs/runs/skill-audit/probe.ps1 +16 -0
- package/docs/runs/skill-audit/probe_classics_branches.ps1 +3 -0
- package/docs/runs/skill-audit/probe_commits.ps1 +15 -0
- package/docs/runs/skill-audit/probe_repos.ps1 +17 -0
- package/docs/runs/skill-audit/probe_sentry_sec.ps1 +3 -0
- package/docs/runs/skill-audit/probe_sentry_sweep.ps1 +5 -0
- package/docs/runs/skill-audit/probe_snyk.ps1 +3 -0
- package/docs/runs/skill-audit/probe_snyk_recipes.ps1 +5 -0
- package/docs/runs/skill-audit/probe_taste_branches.ps1 +8 -0
- package/docs/runs/skill-audit/probe_taste_v19.ps1 +3 -0
- package/docs/runs/skill-audit/probe_user_repos.ps1 +19 -0
- package/docs/runs/skill-audit/prune_taste_skills.cjs +22 -0
- package/docs/runs/skill-audit/run_phase1_2.cjs +142 -0
- package/docs/runs/skill-audit/run_phase6_verification.cjs +107 -0
- package/docs/runs/skill-audit/show_16_details.ps1 +25 -0
- package/docs/runs/skill-audit/show_20_skills.ps1 +28 -0
- package/docs/runs/skill-audit/show_classics_diff.cjs +37 -0
- package/docs/runs/skill-audit/sync_agent_markdowns.cjs +30 -0
- package/docs/runs/skill-audit/taste-skill-upstream.md +542 -0
- package/docs/runs/skill-audit/update-manifest.md +88 -0
- package/docs/runs/skill-audit/update_config_skills.cjs +52 -0
- package/docs/runs/skill-audit/update_registry_phase7.cjs +83 -0
- package/docs/runs/skill-audit/upstream-status.json +563 -0
- package/docs/runs/skill-audit/verification-results.json +252 -0
- package/doctrine/AGENTS.md +39 -77
- package/hooks/git/pre-commit +11 -0
- package/hooks/heimdall-hook.cjs +81 -2
- package/hooks/herdr-agent-state.ps1 +55 -0
- package/hooks/hooks.json +0 -11
- package/lib/catalog-sync.cjs +162 -0
- package/lib/hermes-context.cjs +57 -0
- package/lib/model-context.cjs +123 -0
- package/lib/path-discovery.cjs +94 -0
- package/package.json +7 -3
- package/playbooks/01-interactive-grilling-gate.md +14 -1
- package/playbooks/02-fast-code-deep-verification.md +3 -2
- package/playbooks/03-adversarial-code-review.md +3 -3
- package/playbooks/05-canonical-plan-template.md +32 -18
- package/playbooks/06-model-registry-matrix.md +24 -13
- package/playbooks/07-legacy-code-modernization.md +74 -0
- package/playbooks/08-backlog-and-change-request-flow.md +122 -0
- package/setup.mjs +592 -172
- package/skills/antislop/SKILL.md +76 -56
- package/skills/antislop/VERSION +1 -0
- package/skills/antislop-copywriting/SKILL.md +372 -372
- package/skills/antislop-human/LICENSE-web-quality-skills +21 -0
- package/skills/antislop-human/SKILL.md +204 -147
- package/skills/antislop-human/contrast-check.py +168 -165
- package/skills/antislop-human/contrast-mcp.py +6 -3
- package/skills/antislop-human/references/A11Y-PATTERNS.md +249 -0
- package/skills/antislop-human/references/WCAG.md +127 -0
- package/skills/antislop-ui/LICENSE-web-quality-skills +21 -0
- package/skills/antislop-ui/SKILL.md +344 -304
- package/skills/antislop-ui/references/CORE-WEB-VITALS.md +181 -0
- package/skills/antislop-ui/references/WEB-BASELINE.md +28 -0
- package/skills/brainstorming/SKILL.md +274 -265
- package/skills/brainstorming/builder-check-prompt.md +23 -0
- package/skills/brainstorming/scripts/frame-template.html +213 -213
- package/skills/brainstorming/scripts/server.cjs +723 -723
- package/skills/brandkit/SKILL.md +798 -798
- package/skills/brutalist-skill/SKILL.md +93 -92
- package/skills/caveman/README.md +68 -0
- package/skills/caveman/SKILL.md +73 -38
- package/skills/claude-collaborator/SKILL.md +24 -44
- package/skills/clean-code/SKILL.md +152 -0
- package/skills/cloudflare-security-audit/AI-AND-LLM.md +83 -83
- package/skills/cloudflare-security-audit/ATTACK-CLASSES.md +130 -130
- package/skills/cloudflare-security-audit/CLIENT-SIDE.md +83 -83
- package/skills/cloudflare-security-audit/CLOUD-AND-DEPLOYMENT.md +86 -86
- package/skills/cloudflare-security-audit/DATA-ISOLATION-AND-LIFECYCLE.md +84 -84
- package/skills/cloudflare-security-audit/DESKTOP-MOBILE-AND-LOCAL-IPC.md +89 -89
- package/skills/cloudflare-security-audit/HUNTING.md +251 -251
- package/skills/cloudflare-security-audit/MEMORY-SAFETY-AND-BINARY.md +101 -101
- package/skills/cloudflare-security-audit/PROTOCOLS-RPC-AND-MESSAGING.md +81 -81
- package/skills/cloudflare-security-audit/RECONNAISSANCE.md +156 -156
- package/skills/cloudflare-security-audit/RESOURCE-EXHAUSTION-AND-AVAILABILITY.md +78 -78
- package/skills/cloudflare-security-audit/SKILL.md +192 -192
- package/skills/cloudflare-security-audit/SUPPLY-CHAIN-AND-RELEASE.md +73 -73
- package/skills/cloudflare-security-audit/VALIDATION-AND-REPORTING.md +186 -186
- package/skills/cloudflare-security-audit/WEB-PROTOCOL-AND-AUTH.md +105 -105
- package/skills/cloudflare-security-audit/report-schema.json +461 -461
- package/skills/cloudflare-security-audit/validate-coverage-ledger.cjs +872 -872
- package/skills/cloudflare-security-audit/validate-coverage-ledger.test.cjs +740 -740
- package/skills/cloudflare-security-audit/validate-findings.cjs +773 -773
- package/skills/cloudflare-security-audit/validate-findings.test.cjs +652 -652
- package/skills/codex-collaborator/SKILL.md +24 -45
- package/skills/communication-doctrine/SKILL.md +89 -0
- package/skills/diagnosing-superpowers/SKILL.md +1 -1
- package/skills/diagnosing-superpowers/references/github-issues.md +3 -3
- package/skills/diagnosing-superpowers/templates/issue.md +23 -4
- package/skills/executing-plans/SKILL.md +0 -52
- package/skills/executing-plans/scripts/task-done +52 -52
- package/skills/executing-plans/scripts/task-start +28 -28
- package/skills/gemini-collaborator/SKILL.md +24 -56
- package/skills/herdr/SKILL.md +243 -0
- package/skills/image-to-code-skill/SKILL.md +1228 -1228
- package/skills/imagegen-frontend-mobile/SKILL.md +1465 -1465
- package/skills/imagegen-frontend-web/SKILL.md +987 -987
- package/skills/karpathy-guidelines/SKILL.md +67 -67
- package/skills/mcp-engineering/SKILL.md +74 -0
- package/skills/minimalist-skill/SKILL.md +87 -85
- package/skills/output-skill/SKILL.md +49 -49
- package/skills/ponytail/SKILL.md +36 -52
- package/skills/postgres-best-practices/SKILL.md +30 -52
- package/skills/postgres-best-practices/references/backup-restore.md +372 -0
- package/skills/postgres-best-practices/references/bulk-loading.md +315 -0
- package/skills/postgres-best-practices/references/connection-pooling.md +310 -0
- package/skills/postgres-best-practices/references/hot-standby.md +221 -0
- package/skills/postgres-best-practices/references/indexing.md +318 -0
- package/skills/postgres-best-practices/references/logical-replication.md +610 -0
- package/skills/postgres-best-practices/references/major-version-upgrades.md +334 -0
- package/skills/postgres-best-practices/references/performance-diagnostics.md +553 -0
- package/skills/postgres-best-practices/references/query-optimization.md +294 -0
- package/skills/postgres-best-practices/references/query-patterns.md +566 -0
- package/skills/postgres-best-practices/references/schema-design.md +362 -0
- package/skills/postgres-best-practices/references/security-roles.md +489 -0
- package/skills/postgres-best-practices/references/transaction-isolation.md +344 -0
- package/skills/progressive-disclosure/SKILL.md +68 -0
- package/skills/pstack-discipline/SKILL.md +40 -0
- package/skills/receiving-code-review/SKILL.md +2 -23
- package/skills/requesting-code-review/SKILL.md +7 -36
- package/skills/soft-skill/SKILL.md +99 -98
- package/skills/stitch-skill/DESIGN.md +121 -121
- package/skills/stitch-skill/SKILL.md +185 -184
- package/skills/strix-security/SKILL.md +54 -53
- package/skills/subagent-driven-development/SKILL.md +28 -260
- package/skills/subagent-driven-development/references/batching.md +11 -0
- package/skills/subagent-driven-development/references/cannot-verify.md +10 -0
- package/skills/subagent-driven-development/references/final-review.md +47 -0
- package/skills/subagent-driven-development/references/fix-loop.md +77 -0
- package/skills/subagent-driven-development/references/handling-reports.md +15 -0
- package/skills/subagent-driven-development/references/preflight.md +26 -0
- package/skills/subagent-driven-development/references/resuming.md +12 -0
- package/skills/subagent-driven-development/scripts/review-package +53 -53
- package/skills/subagent-driven-development/scripts/sdd-workspace +87 -82
- package/skills/subagent-driven-development/scripts/task-brief +43 -43
- package/skills/systematic-debugging/SKILL.md +283 -306
- package/skills/taste-skill/SKILL.md +542 -1206
- package/skills/taste-skill/references/app-screens.md +1199 -0
- package/skills/taste-skill/references/colour.md +141 -0
- package/skills/taste-skill/references/copy.md +177 -0
- package/skills/taste-skill/references/critique.md +182 -0
- package/skills/taste-skill/references/imagery.md +791 -0
- package/skills/taste-skill/references/layout.md +292 -0
- package/skills/taste-skill/references/motion.md +980 -0
- package/skills/taste-skill/references/research.md +356 -0
- package/skills/taste-skill/references/scenes.md +174 -0
- package/skills/taste-skill/references/tells.md +115 -0
- package/skills/taste-skill/references/type.md +170 -0
- package/skills/test-driven-development/SKILL.md +330 -337
- package/skills/thermo/SKILL.md +67 -0
- package/skills/threat-modeling/SKILL.md +55 -0
- package/skills/trailofbits-audit-context/SKILL.md +72 -72
- package/skills/trailofbits-insecure-defaults/commands/audit.md +35 -0
- package/skills/trailofbits-insecure-defaults/references/debug-features.json +13 -13
- package/skills/trailofbits-insecure-defaults/references/debug-features.md +75 -75
- package/skills/trailofbits-insecure-defaults/references/default-credentials.json +12 -12
- package/skills/trailofbits-insecure-defaults/references/default-credentials.md +76 -76
- package/skills/trailofbits-insecure-defaults/references/fail-open-security.json +11 -11
- package/skills/trailofbits-insecure-defaults/references/fail-open-security.md +70 -70
- package/skills/trailofbits-insecure-defaults/references/fallback-secrets.json +11 -11
- package/skills/trailofbits-insecure-defaults/references/fallback-secrets.md +70 -70
- package/skills/trailofbits-insecure-defaults/references/permissive-access.json +13 -13
- package/skills/trailofbits-insecure-defaults/references/permissive-access.md +62 -62
- package/skills/trailofbits-insecure-defaults/references/weak-crypto.json +13 -13
- package/skills/trailofbits-insecure-defaults/references/weak-crypto.md +71 -71
- package/skills/trailofbits-triage-brocards/SKILL.md +208 -208
- package/skills/verification-before-completion/SKILL.md +0 -14
- package/skills/writing-plans/SKILL.md +11 -15
- package/skills/writing-skills/SKILL.md +21 -303
- package/skills/writing-skills/references/CLAUDE_MD_TESTING.md +189 -0
- package/skills/writing-skills/references/anthropic-best-practices.md +1150 -0
- package/skills/writing-skills/references/bulletproofing.md +80 -0
- package/skills/writing-skills/references/discovery.md +201 -0
- package/skills/writing-skills/references/flowcharts.md +37 -0
- package/skills/writing-skills/references/graphviz-conventions.dot +172 -0
- package/skills/writing-skills/references/persuasion-principles.md +187 -0
- package/skills/writing-skills/references/testing-skills-with-subagents.md +384 -0
- package/tools/{claude-bridge.ps1 → archive/claude-bridge.ps1} +10 -7
- package/tools/{codex-bridge.ps1 → archive/codex-bridge.ps1} +1 -1
- package/tools/{gemini-bridge.ps1 → archive/gemini-bridge.ps1} +12 -19
- package/skills/gpt-tasteskill/SKILL.md +0 -74
- package/skills/taste-skill-v1/SKILL.md +0 -226
- /package/tools/{claude-bridge.cmd → archive/claude-bridge.cmd} +0 -0
- /package/tools/{claude-bridge.sh → archive/claude-bridge.sh} +0 -0
- /package/tools/{codex-bridge.cmd → archive/codex-bridge.cmd} +0 -0
- /package/tools/{codex-bridge.sh → archive/codex-bridge.sh} +0 -0
- /package/tools/{codexcli-bridge.cmd → archive/codexcli-bridge.cmd} +0 -0
- /package/tools/{codexcli-bridge.ps1 → archive/codexcli-bridge.ps1} +0 -0
- /package/tools/{gemini-bridge.cmd → archive/gemini-bridge.cmd} +0 -0
- /package/tools/{gemini-bridge.sh → archive/gemini-bridge.sh} +0 -0
|
@@ -0,0 +1,315 @@
|
|
|
1
|
+
# Bulk Data Loading Reference
|
|
2
|
+
|
|
3
|
+
## Contents
|
|
4
|
+
- COPY vs INSERT performance
|
|
5
|
+
- COPY FROM patterns
|
|
6
|
+
- Optimizing large loads
|
|
7
|
+
- ETL staging patterns
|
|
8
|
+
- Bulk updates and deletes
|
|
9
|
+
|
|
10
|
+
## COPY vs INSERT Performance
|
|
11
|
+
|
|
12
|
+
| Method | Rows/sec (typical) | Use case |
|
|
13
|
+
|--------|-------------------|----------|
|
|
14
|
+
| Single-row INSERT | ~1,000-5,000 | Application writes |
|
|
15
|
+
| Multi-row INSERT (VALUES) | ~10,000-50,000 | Batch inserts from code |
|
|
16
|
+
| `COPY FROM` | ~100,000-500,000+ | Bulk loading from files or streams |
|
|
17
|
+
| `COPY FROM` (binary) | ~200,000-1,000,000+ | Maximum throughput (binary format) |
|
|
18
|
+
|
|
19
|
+
`COPY` is **10-100x faster** than individual INSERTs because it bypasses per-row overhead (parsing, planning, WAL per statement).
|
|
20
|
+
|
|
21
|
+
## COPY FROM Patterns
|
|
22
|
+
|
|
23
|
+
### From a File
|
|
24
|
+
|
|
25
|
+
```sql
|
|
26
|
+
-- CSV with header
|
|
27
|
+
COPY orders FROM '/path/to/orders.csv' WITH (FORMAT csv, HEADER true);
|
|
28
|
+
|
|
29
|
+
-- Tab-delimited
|
|
30
|
+
COPY orders FROM '/path/to/orders.tsv' WITH (FORMAT text);
|
|
31
|
+
|
|
32
|
+
-- Custom delimiter
|
|
33
|
+
COPY orders FROM '/path/to/orders.dat' WITH (DELIMITER '|');
|
|
34
|
+
|
|
35
|
+
-- With NULL handling
|
|
36
|
+
COPY orders FROM '/path/to/orders.csv'
|
|
37
|
+
WITH (FORMAT csv, HEADER true, NULL '');
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
### From STDIN (Piped Data)
|
|
41
|
+
|
|
42
|
+
```bash
|
|
43
|
+
# Pipe from another command
|
|
44
|
+
cat orders.csv | psql -d mydb -c "\COPY orders FROM STDIN WITH (FORMAT csv, HEADER true)"
|
|
45
|
+
|
|
46
|
+
# Pipe from gzip
|
|
47
|
+
gunzip -c orders.csv.gz | psql -d mydb -c "\COPY orders FROM STDIN WITH (FORMAT csv, HEADER true)"
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
### From Application Code
|
|
51
|
+
|
|
52
|
+
Most drivers support COPY protocol for streaming data:
|
|
53
|
+
|
|
54
|
+
```python
|
|
55
|
+
# Python (psycopg 3)
|
|
56
|
+
with conn.cursor() as cur:
|
|
57
|
+
with cur.copy("COPY orders (id, customer_id, total) FROM STDIN") as copy:
|
|
58
|
+
for row in data:
|
|
59
|
+
copy.write_row(row)
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
```javascript
|
|
63
|
+
// Node.js (pg-copy-streams)
|
|
64
|
+
const { from } = require('pg-copy-streams');
|
|
65
|
+
const stream = client.query(from('COPY orders FROM STDIN WITH (FORMAT csv)'));
|
|
66
|
+
fileStream.pipe(stream);
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
### COPY TO (Export)
|
|
70
|
+
|
|
71
|
+
```sql
|
|
72
|
+
-- Export to CSV
|
|
73
|
+
COPY orders TO '/path/to/orders.csv' WITH (FORMAT csv, HEADER true);
|
|
74
|
+
|
|
75
|
+
-- Export query results
|
|
76
|
+
COPY (SELECT id, total FROM orders WHERE created_at > '2024-01-01')
|
|
77
|
+
TO '/path/to/recent.csv' WITH (FORMAT csv, HEADER true);
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
### Error Handling (PG17+)
|
|
81
|
+
|
|
82
|
+
```sql
|
|
83
|
+
-- Skip invalid rows instead of failing the entire COPY
|
|
84
|
+
COPY orders FROM '/path/to/orders.csv'
|
|
85
|
+
WITH (FORMAT csv, HEADER true, ON_ERROR ignore);
|
|
86
|
+
|
|
87
|
+
-- PG18+: limit how many errors to tolerate
|
|
88
|
+
COPY orders FROM '/path/to/orders.csv'
|
|
89
|
+
WITH (FORMAT csv, HEADER true, ON_ERROR ignore, REJECT_LIMIT 100);
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
## Optimizing Large Loads
|
|
93
|
+
|
|
94
|
+
### 1. Drop Indexes, Load, Recreate
|
|
95
|
+
|
|
96
|
+
Building indexes incrementally during COPY is much slower than building them once after:
|
|
97
|
+
|
|
98
|
+
```sql
|
|
99
|
+
-- Before load: drop non-essential indexes
|
|
100
|
+
DROP INDEX idx_orders_customer_id;
|
|
101
|
+
DROP INDEX idx_orders_created_at;
|
|
102
|
+
|
|
103
|
+
-- Load data
|
|
104
|
+
COPY orders FROM '/path/to/orders.csv' WITH (FORMAT csv, HEADER true);
|
|
105
|
+
|
|
106
|
+
-- After load: recreate indexes (concurrently if table is in use)
|
|
107
|
+
CREATE INDEX idx_orders_customer_id ON orders(customer_id);
|
|
108
|
+
CREATE INDEX idx_orders_created_at ON orders(created_at);
|
|
109
|
+
|
|
110
|
+
-- Update statistics
|
|
111
|
+
ANALYZE orders;
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
### 2. Disable Triggers During Load
|
|
115
|
+
|
|
116
|
+
```sql
|
|
117
|
+
-- Disable all triggers on the table
|
|
118
|
+
ALTER TABLE orders DISABLE TRIGGER ALL;
|
|
119
|
+
|
|
120
|
+
-- Load data
|
|
121
|
+
COPY orders FROM '/path/to/orders.csv' WITH (FORMAT csv, HEADER true);
|
|
122
|
+
|
|
123
|
+
-- Re-enable triggers
|
|
124
|
+
ALTER TABLE orders ENABLE TRIGGER ALL;
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
`DISABLE TRIGGER ALL` requires superuser privileges when the table has foreign-key or other internally generated constraint triggers. A table owner can use `DISABLE TRIGGER USER` to disable only user-defined triggers. Disabling constraint triggers can admit invalid data, so validate constraints before re-enabling writes.
|
|
128
|
+
|
|
129
|
+
### 3. Increase maintenance_work_mem
|
|
130
|
+
|
|
131
|
+
Larger `maintenance_work_mem` speeds up index creation after the load:
|
|
132
|
+
|
|
133
|
+
```sql
|
|
134
|
+
SET maintenance_work_mem = '1GB'; -- for the duration of the load session
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
### 4. Disable Autovacuum During Load
|
|
138
|
+
|
|
139
|
+
For very large bulk loads, temporarily disable autovacuum to avoid competing I/O:
|
|
140
|
+
|
|
141
|
+
```sql
|
|
142
|
+
ALTER TABLE orders SET (autovacuum_enabled = false);
|
|
143
|
+
|
|
144
|
+
-- Load data...
|
|
145
|
+
|
|
146
|
+
ALTER TABLE orders SET (autovacuum_enabled = true);
|
|
147
|
+
VACUUM ANALYZE orders;
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
### 5. Use Unlogged Tables for Staging
|
|
151
|
+
|
|
152
|
+
Unlogged tables skip WAL writes — 2-3x faster for writes but **data is lost on crash**:
|
|
153
|
+
|
|
154
|
+
```sql
|
|
155
|
+
CREATE UNLOGGED TABLE staging_orders (LIKE orders INCLUDING ALL);
|
|
156
|
+
|
|
157
|
+
COPY staging_orders FROM '/path/to/orders.csv' WITH (FORMAT csv, HEADER true);
|
|
158
|
+
|
|
159
|
+
-- Transform and move to the real table
|
|
160
|
+
INSERT INTO orders SELECT * FROM staging_orders;
|
|
161
|
+
|
|
162
|
+
DROP TABLE staging_orders;
|
|
163
|
+
```
|
|
164
|
+
|
|
165
|
+
### 6. Batch Size for Programmatic Inserts
|
|
166
|
+
|
|
167
|
+
When COPY isn't available, use multi-row INSERT with batches of 100-1,000 rows:
|
|
168
|
+
|
|
169
|
+
```sql
|
|
170
|
+
-- Single round-trip for 1,000 rows
|
|
171
|
+
INSERT INTO orders (customer_id, total, created_at)
|
|
172
|
+
VALUES
|
|
173
|
+
(1, 99.99, now()),
|
|
174
|
+
(2, 149.99, now()),
|
|
175
|
+
-- ... up to ~1,000 rows per statement
|
|
176
|
+
(1000, 79.99, now());
|
|
177
|
+
```
|
|
178
|
+
|
|
179
|
+
Beyond ~1,000 rows per statement, parse overhead increases. Use COPY for larger batches.
|
|
180
|
+
|
|
181
|
+
### 7. Parallel Loading into Partitioned Tables
|
|
182
|
+
|
|
183
|
+
Load data into individual partitions concurrently from separate sessions:
|
|
184
|
+
|
|
185
|
+
```bash
|
|
186
|
+
# Session 1
|
|
187
|
+
psql -c "COPY events_2024_q1 FROM '/data/q1.csv' WITH (FORMAT csv)"
|
|
188
|
+
|
|
189
|
+
# Session 2 (concurrent)
|
|
190
|
+
psql -c "COPY events_2024_q2 FROM '/data/q2.csv' WITH (FORMAT csv)"
|
|
191
|
+
```
|
|
192
|
+
|
|
193
|
+
## ETL Staging Patterns
|
|
194
|
+
|
|
195
|
+
### Staging Table with Upsert
|
|
196
|
+
|
|
197
|
+
```sql
|
|
198
|
+
-- Create staging table (temporary or unlogged)
|
|
199
|
+
CREATE TEMP TABLE staging_customers (LIKE customers INCLUDING DEFAULTS);
|
|
200
|
+
|
|
201
|
+
-- Load raw data
|
|
202
|
+
COPY staging_customers FROM '/path/to/customers.csv' WITH (FORMAT csv, HEADER true);
|
|
203
|
+
|
|
204
|
+
-- Upsert into production table
|
|
205
|
+
INSERT INTO customers (id, name, email, updated_at)
|
|
206
|
+
SELECT id, name, email, now()
|
|
207
|
+
FROM staging_customers
|
|
208
|
+
ON CONFLICT (id)
|
|
209
|
+
DO UPDATE SET
|
|
210
|
+
name = EXCLUDED.name,
|
|
211
|
+
email = EXCLUDED.email,
|
|
212
|
+
updated_at = EXCLUDED.updated_at;
|
|
213
|
+
```
|
|
214
|
+
|
|
215
|
+
### Staging Table with MERGE (PG15+)
|
|
216
|
+
|
|
217
|
+
```sql
|
|
218
|
+
MERGE INTO customers AS target
|
|
219
|
+
USING staging_customers AS source
|
|
220
|
+
ON target.id = source.id
|
|
221
|
+
WHEN MATCHED AND source.name IS DISTINCT FROM target.name THEN
|
|
222
|
+
UPDATE SET name = source.name, email = source.email, updated_at = now()
|
|
223
|
+
WHEN NOT MATCHED THEN
|
|
224
|
+
INSERT (id, name, email, updated_at)
|
|
225
|
+
VALUES (source.id, source.name, source.email, now());
|
|
226
|
+
```
|
|
227
|
+
|
|
228
|
+
### Swap Table Pattern
|
|
229
|
+
|
|
230
|
+
For full-refresh loads where you replace all data:
|
|
231
|
+
|
|
232
|
+
```sql
|
|
233
|
+
-- Load into a new table
|
|
234
|
+
CREATE TABLE orders_new (LIKE orders INCLUDING ALL);
|
|
235
|
+
COPY orders_new FROM '/path/to/orders.csv' WITH (FORMAT csv, HEADER true);
|
|
236
|
+
ANALYZE orders_new;
|
|
237
|
+
|
|
238
|
+
-- Atomic swap (brief exclusive lock)
|
|
239
|
+
BEGIN;
|
|
240
|
+
ALTER TABLE orders RENAME TO orders_old;
|
|
241
|
+
ALTER TABLE orders_new RENAME TO orders;
|
|
242
|
+
DROP TABLE orders_old;
|
|
243
|
+
COMMIT;
|
|
244
|
+
```
|
|
245
|
+
|
|
246
|
+
`LIKE ... INCLUDING ALL` does not copy foreign keys, triggers, rules, grants, row-level security policies, or publication membership. Recreate and verify those objects before swapping, or use a data-only refresh when the original table's identity and dependencies must remain unchanged.
|
|
247
|
+
|
|
248
|
+
## Bulk Updates and Deletes
|
|
249
|
+
|
|
250
|
+
### Chunked Deletes
|
|
251
|
+
|
|
252
|
+
Large DELETEs lock rows and generate WAL. This loop limits each statement to 10,000 rows:
|
|
253
|
+
|
|
254
|
+
```sql
|
|
255
|
+
-- Delete in batches of 10,000
|
|
256
|
+
DO $$
|
|
257
|
+
DECLARE
|
|
258
|
+
rows_deleted int;
|
|
259
|
+
BEGIN
|
|
260
|
+
LOOP
|
|
261
|
+
DELETE FROM audit_log
|
|
262
|
+
WHERE id IN (
|
|
263
|
+
SELECT id FROM audit_log
|
|
264
|
+
WHERE created_at < now() - interval '1 year'
|
|
265
|
+
LIMIT 10000
|
|
266
|
+
);
|
|
267
|
+
GET DIAGNOSTICS rows_deleted = ROW_COUNT;
|
|
268
|
+
EXIT WHEN rows_deleted = 0;
|
|
269
|
+
-- Optional: brief pause to reduce WAL pressure
|
|
270
|
+
PERFORM pg_sleep(0.1);
|
|
271
|
+
END LOOP;
|
|
272
|
+
END $$;
|
|
273
|
+
```
|
|
274
|
+
|
|
275
|
+
The entire `DO` block is still one transaction: locks, WAL, and dead rows accumulate until it finishes. For true transaction-level batching, execute one limited DELETE per client transaction and commit between batches.
|
|
276
|
+
|
|
277
|
+
### Chunked Updates
|
|
278
|
+
|
|
279
|
+
The same statement-size pattern works for large updates:
|
|
280
|
+
|
|
281
|
+
```sql
|
|
282
|
+
DO $$
|
|
283
|
+
DECLARE
|
|
284
|
+
rows_updated int;
|
|
285
|
+
BEGIN
|
|
286
|
+
LOOP
|
|
287
|
+
UPDATE orders
|
|
288
|
+
SET status = 'archived'
|
|
289
|
+
WHERE id IN (
|
|
290
|
+
SELECT id FROM orders
|
|
291
|
+
WHERE status = 'completed'
|
|
292
|
+
AND created_at < now() - interval '2 years'
|
|
293
|
+
LIMIT 10000
|
|
294
|
+
FOR UPDATE SKIP LOCKED
|
|
295
|
+
);
|
|
296
|
+
GET DIAGNOSTICS rows_updated = ROW_COUNT;
|
|
297
|
+
EXIT WHEN rows_updated = 0;
|
|
298
|
+
END LOOP;
|
|
299
|
+
END $$;
|
|
300
|
+
```
|
|
301
|
+
|
|
302
|
+
As with the DELETE loop, the `DO` block commits only once. Drive batches from the client or a transaction-controlling procedure invoked outside an explicit transaction when each batch must commit independently.
|
|
303
|
+
|
|
304
|
+
### Partition Drop Instead of Delete
|
|
305
|
+
|
|
306
|
+
If data is partitioned by time, dropping a partition is instant vs. a slow DELETE:
|
|
307
|
+
|
|
308
|
+
```sql
|
|
309
|
+
-- Instant: drop old partition
|
|
310
|
+
ALTER TABLE events DETACH PARTITION events_2023_q1 CONCURRENTLY;
|
|
311
|
+
DROP TABLE events_2023_q1;
|
|
312
|
+
|
|
313
|
+
-- vs. slow: delete rows
|
|
314
|
+
-- DELETE FROM events WHERE occurred_at < '2024-01-01'; -- don't do this
|
|
315
|
+
```
|
|
@@ -0,0 +1,310 @@
|
|
|
1
|
+
# Connection Pooling Reference
|
|
2
|
+
|
|
3
|
+
## Contents
|
|
4
|
+
- Why connection pooling matters
|
|
5
|
+
- PgBouncer configuration
|
|
6
|
+
- Pool modes
|
|
7
|
+
- Prepared statement handling
|
|
8
|
+
- Monitoring and diagnostics
|
|
9
|
+
- Application-side pooling
|
|
10
|
+
|
|
11
|
+
## Why Connection Pooling Matters
|
|
12
|
+
|
|
13
|
+
Each PostgreSQL connection spawns a dedicated backend process (~5-10 MB of memory). Without pooling:
|
|
14
|
+
|
|
15
|
+
- 500 application instances × 10 connections each = 5,000 backend processes
|
|
16
|
+
- Memory: 5,000 × 10 MB = ~50 GB just for connection overhead
|
|
17
|
+
- Context switching degrades performance above a few hundred active backends
|
|
18
|
+
- `max_connections` must be set high, wasting shared memory
|
|
19
|
+
|
|
20
|
+
A connection pooler sits between the application and PostgreSQL, multiplexing many client connections onto fewer server connections.
|
|
21
|
+
|
|
22
|
+
### Connection Overhead
|
|
23
|
+
|
|
24
|
+
```sql
|
|
25
|
+
-- Current connections vs. limit
|
|
26
|
+
SELECT
|
|
27
|
+
count(*) AS current,
|
|
28
|
+
(SELECT setting::int FROM pg_settings WHERE name = 'max_connections') AS max,
|
|
29
|
+
(SELECT setting::int FROM pg_settings WHERE name = 'superuser_reserved_connections') AS reserved
|
|
30
|
+
FROM pg_stat_activity;
|
|
31
|
+
|
|
32
|
+
-- Potential per-operation and per-session memory settings
|
|
33
|
+
SELECT
|
|
34
|
+
name,
|
|
35
|
+
current_setting(name) AS configured_value,
|
|
36
|
+
pg_size_bytes(current_setting(name)) AS configured_bytes
|
|
37
|
+
FROM pg_settings
|
|
38
|
+
WHERE name IN ('work_mem', 'temp_buffers')
|
|
39
|
+
ORDER BY name;
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
These are not baseline allocations per backend. `work_mem` can be consumed by multiple query operations, while `temp_buffers` is allocated lazily when a session uses temporary tables.
|
|
43
|
+
|
|
44
|
+
## PgBouncer Configuration
|
|
45
|
+
|
|
46
|
+
PgBouncer is the most widely used connection pooler for PostgreSQL.
|
|
47
|
+
|
|
48
|
+
### Essential pgbouncer.ini Settings
|
|
49
|
+
|
|
50
|
+
```ini
|
|
51
|
+
[databases]
|
|
52
|
+
mydb = host=127.0.0.1 port=5432 dbname=mydb
|
|
53
|
+
|
|
54
|
+
[pgbouncer]
|
|
55
|
+
listen_addr = 0.0.0.0
|
|
56
|
+
listen_port = 6432
|
|
57
|
+
auth_type = scram-sha-256
|
|
58
|
+
auth_file = /etc/pgbouncer/userlist.txt
|
|
59
|
+
|
|
60
|
+
# Pool sizing
|
|
61
|
+
pool_mode = transaction
|
|
62
|
+
default_pool_size = 20
|
|
63
|
+
min_pool_size = 5
|
|
64
|
+
max_client_conn = 1000
|
|
65
|
+
max_db_connections = 50
|
|
66
|
+
|
|
67
|
+
# Timeouts
|
|
68
|
+
server_idle_timeout = 300
|
|
69
|
+
client_idle_timeout = 0
|
|
70
|
+
query_timeout = 0
|
|
71
|
+
query_wait_timeout = 120
|
|
72
|
+
server_login_retry = 15
|
|
73
|
+
|
|
74
|
+
# Logging
|
|
75
|
+
log_connections = 0
|
|
76
|
+
log_disconnections = 0
|
|
77
|
+
stats_period = 60
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
### Key Parameters
|
|
81
|
+
|
|
82
|
+
| Parameter | Recommended | Purpose |
|
|
83
|
+
|-----------|------------|---------|
|
|
84
|
+
| `default_pool_size` | 20-50 | Server connections per user/database pair |
|
|
85
|
+
| `min_pool_size` | 5 | Minimum idle server connections to keep open |
|
|
86
|
+
| `max_client_conn` | 1000-10000 | Max client connections PgBouncer accepts |
|
|
87
|
+
| `max_db_connections` | 50-100 | Hard cap on server connections per database |
|
|
88
|
+
| `reserve_pool_size` | 5 | Extra connections for burst traffic |
|
|
89
|
+
| `reserve_pool_timeout` | 3 | Seconds before using reserve pool |
|
|
90
|
+
| `query_wait_timeout` | 120 | Max time a client waits for a server connection |
|
|
91
|
+
|
|
92
|
+
### Sizing Rule of Thumb
|
|
93
|
+
|
|
94
|
+
**Server connections**: Set `default_pool_size` to roughly 2-4x the number of CPU cores on the database server. More connections don't help — they just increase lock contention and context switching.
|
|
95
|
+
|
|
96
|
+
**Client connections**: Set `max_client_conn` high enough for all application instances. PgBouncer handles thousands of idle client connections with minimal memory.
|
|
97
|
+
|
|
98
|
+
```
|
|
99
|
+
App instances (500 clients) ──→ PgBouncer (max_client_conn=1000) ──→ PostgreSQL (pool_size=20)
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
## Pool Modes
|
|
103
|
+
|
|
104
|
+
### Transaction Mode (Recommended for Most Applications)
|
|
105
|
+
|
|
106
|
+
```ini
|
|
107
|
+
pool_mode = transaction
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
Server connection is assigned when a transaction begins and returned when it commits/rolls back. Between transactions, the connection is available to other clients.
|
|
111
|
+
|
|
112
|
+
**Compatible with**: Standard SQL, parameterized queries, most ORMs.
|
|
113
|
+
|
|
114
|
+
**NOT compatible with** (session-level features):
|
|
115
|
+
- `SET` / `RESET` (use `SET LOCAL` inside a transaction instead)
|
|
116
|
+
- `LISTEN` / `NOTIFY`
|
|
117
|
+
- SQL-level `PREPARE` / `DEALLOCATE` (use protocol-level prepared statements)
|
|
118
|
+
- `DECLARE ... WITH HOLD` cursors
|
|
119
|
+
- Temporary tables with `ON COMMIT PRESERVE ROWS`
|
|
120
|
+
- Session-level advisory locks (`pg_advisory_lock` — use `pg_advisory_xact_lock` instead)
|
|
121
|
+
- `LOAD` statement
|
|
122
|
+
|
|
123
|
+
### Session Mode
|
|
124
|
+
|
|
125
|
+
```ini
|
|
126
|
+
pool_mode = session
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
Server connection is held for the entire client session. Compatible with all PostgreSQL features but offers less multiplexing benefit.
|
|
130
|
+
|
|
131
|
+
**Use when**: Application relies on session-level features (temp tables, LISTEN/NOTIFY, SET parameters).
|
|
132
|
+
|
|
133
|
+
### Statement Mode
|
|
134
|
+
|
|
135
|
+
```ini
|
|
136
|
+
pool_mode = statement
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
Server connection released after every statement. Maximum multiplexing but **incompatible with multi-statement transactions**.
|
|
140
|
+
|
|
141
|
+
**Use when**: Application only runs autocommit single statements (rare).
|
|
142
|
+
|
|
143
|
+
### Choosing a Mode
|
|
144
|
+
|
|
145
|
+
| Application pattern | Recommended mode |
|
|
146
|
+
|--------------------|-----------------|
|
|
147
|
+
| Web apps, APIs, serverless | Transaction |
|
|
148
|
+
| Applications using LISTEN/NOTIFY | Session |
|
|
149
|
+
| Applications using temp tables across transactions | Session |
|
|
150
|
+
| Applications with SET session variables | Session (or refactor to `SET LOCAL`) |
|
|
151
|
+
| Simple autocommit queries | Statement |
|
|
152
|
+
|
|
153
|
+
## Prepared Statement Handling
|
|
154
|
+
|
|
155
|
+
### The Problem
|
|
156
|
+
|
|
157
|
+
SQL-level `PREPARE`/`EXECUTE` creates server-side prepared statements tied to a session. In transaction mode, the next transaction may use a different server connection where the prepared statement doesn't exist.
|
|
158
|
+
|
|
159
|
+
### Solutions
|
|
160
|
+
|
|
161
|
+
**Protocol-level prepared statements**: Most modern drivers use the PostgreSQL wire protocol's `Parse`/`Bind`/`Execute` messages, which PgBouncer (1.21+) can handle:
|
|
162
|
+
|
|
163
|
+
```ini
|
|
164
|
+
# PgBouncer 1.21+
|
|
165
|
+
max_prepared_statements = 100 # per server connection
|
|
166
|
+
```
|
|
167
|
+
|
|
168
|
+
**Driver-level configuration** to avoid SQL-level PREPARE:
|
|
169
|
+
|
|
170
|
+
```python
|
|
171
|
+
# Python psycopg: uses protocol-level by default — no change needed
|
|
172
|
+
```
|
|
173
|
+
|
|
174
|
+
```javascript
|
|
175
|
+
// Node.js pg: protocol-level by default
|
|
176
|
+
const pool = new Pool({ ...config });
|
|
177
|
+
// For explicit control:
|
|
178
|
+
// statement_timeout via SET LOCAL, not SET
|
|
179
|
+
```
|
|
180
|
+
|
|
181
|
+
```java
|
|
182
|
+
// JDBC: protocol-level by default with prepareThreshold
|
|
183
|
+
// ?prepareThreshold=5 (default: 5 uses before server-side prepare)
|
|
184
|
+
// ?prepareThreshold=0 (disable server-side prepare entirely)
|
|
185
|
+
```
|
|
186
|
+
|
|
187
|
+
```ruby
|
|
188
|
+
# Ruby pg: protocol-level by default
|
|
189
|
+
# ActiveRecord: no special config needed
|
|
190
|
+
```
|
|
191
|
+
|
|
192
|
+
## Monitoring and Diagnostics
|
|
193
|
+
|
|
194
|
+
### PgBouncer Admin Console
|
|
195
|
+
|
|
196
|
+
Connect to PgBouncer's admin port:
|
|
197
|
+
|
|
198
|
+
```bash
|
|
199
|
+
psql -h 127.0.0.1 -p 6432 -U pgbouncer pgbouncer
|
|
200
|
+
```
|
|
201
|
+
|
|
202
|
+
### Key Commands
|
|
203
|
+
|
|
204
|
+
```text
|
|
205
|
+
-- Pool status (most useful)
|
|
206
|
+
SHOW POOLS;
|
|
207
|
+
-- Columns: database, user, cl_active, cl_waiting, sv_active, sv_idle, sv_used, pool_mode
|
|
208
|
+
|
|
209
|
+
-- Active client and server connections
|
|
210
|
+
SHOW CLIENTS;
|
|
211
|
+
SHOW SERVERS;
|
|
212
|
+
|
|
213
|
+
-- Aggregate statistics
|
|
214
|
+
SHOW STATS;
|
|
215
|
+
-- Columns: total_xact_count, total_query_count, avg_xact_time, avg_query_time
|
|
216
|
+
|
|
217
|
+
-- Configuration
|
|
218
|
+
SHOW CONFIG;
|
|
219
|
+
|
|
220
|
+
-- Memory usage
|
|
221
|
+
SHOW MEM;
|
|
222
|
+
```
|
|
223
|
+
|
|
224
|
+
These commands use PgBouncer's admin protocol and fail if sent directly to PostgreSQL.
|
|
225
|
+
|
|
226
|
+
### What to Watch
|
|
227
|
+
|
|
228
|
+
| Metric | Healthy | Problem |
|
|
229
|
+
|--------|---------|---------|
|
|
230
|
+
| `cl_waiting` | 0 | > 0 = clients waiting for server connections |
|
|
231
|
+
| `sv_active` | < pool_size | = pool_size = pool exhausted |
|
|
232
|
+
| `sv_idle` | > 0 | 0 = no spare connections |
|
|
233
|
+
| `avg_xact_time` | < 100ms | High = long transactions hogging connections |
|
|
234
|
+
| `avg_wait_time` | 0 | > 0 = pool too small or transactions too long |
|
|
235
|
+
|
|
236
|
+
### Common Issues
|
|
237
|
+
|
|
238
|
+
**Clients waiting (`cl_waiting > 0`)**:
|
|
239
|
+
1. Check for long-running transactions: `SELECT * FROM pg_stat_activity WHERE state = 'idle in transaction';`
|
|
240
|
+
2. Increase `default_pool_size` (but diminishing returns beyond ~4x CPU cores)
|
|
241
|
+
3. Check if `query_wait_timeout` is being hit (errors in application logs)
|
|
242
|
+
|
|
243
|
+
**Server connections not being returned**:
|
|
244
|
+
1. `idle in transaction` sessions hold connections — set `idle_in_transaction_session_timeout` in PostgreSQL
|
|
245
|
+
2. Application not committing/rolling back — add explicit transaction management
|
|
246
|
+
|
|
247
|
+
## Application-Side Pooling
|
|
248
|
+
|
|
249
|
+
When PgBouncer isn't available, most drivers offer built-in connection pooling.
|
|
250
|
+
|
|
251
|
+
### Recommended Pool Sizes
|
|
252
|
+
|
|
253
|
+
**Per application instance**: 5-20 connections. Start low, increase only if you see connection wait times.
|
|
254
|
+
|
|
255
|
+
**Total across all instances**: Should not exceed ~4x database CPU cores for active connections.
|
|
256
|
+
|
|
257
|
+
### Common Driver Configuration
|
|
258
|
+
|
|
259
|
+
```python
|
|
260
|
+
# Python (psycopg pool)
|
|
261
|
+
from psycopg_pool import ConnectionPool
|
|
262
|
+
|
|
263
|
+
pool = ConnectionPool(
|
|
264
|
+
conninfo="host=db port=5432 dbname=mydb",
|
|
265
|
+
min_size=5,
|
|
266
|
+
max_size=20,
|
|
267
|
+
max_idle=300, # close idle connections after 5 min
|
|
268
|
+
)
|
|
269
|
+
|
|
270
|
+
with pool.connection() as conn:
|
|
271
|
+
conn.execute("SELECT ...")
|
|
272
|
+
```
|
|
273
|
+
|
|
274
|
+
```javascript
|
|
275
|
+
// Node.js (pg)
|
|
276
|
+
const { Pool } = require('pg');
|
|
277
|
+
const pool = new Pool({
|
|
278
|
+
host: 'db',
|
|
279
|
+
database: 'mydb',
|
|
280
|
+
max: 20, // max connections in pool
|
|
281
|
+
idleTimeoutMillis: 30000,
|
|
282
|
+
connectionTimeoutMillis: 5000,
|
|
283
|
+
});
|
|
284
|
+
|
|
285
|
+
const result = await pool.query('SELECT ...');
|
|
286
|
+
```
|
|
287
|
+
|
|
288
|
+
```java
|
|
289
|
+
// Java (HikariCP)
|
|
290
|
+
HikariConfig config = new HikariConfig();
|
|
291
|
+
config.setJdbcUrl("jdbc:postgresql://db:5432/mydb");
|
|
292
|
+
config.setMaximumPoolSize(20);
|
|
293
|
+
config.setMinimumIdle(5);
|
|
294
|
+
config.setIdleTimeout(300000);
|
|
295
|
+
config.setConnectionTimeout(5000);
|
|
296
|
+
|
|
297
|
+
HikariDataSource ds = new HikariDataSource(config);
|
|
298
|
+
```
|
|
299
|
+
|
|
300
|
+
### Application Pool + PgBouncer
|
|
301
|
+
|
|
302
|
+
You can stack both. Each application instance pools locally (5-10 connections), and PgBouncer pools across all instances:
|
|
303
|
+
|
|
304
|
+
```
|
|
305
|
+
App1 (pool: 10) ──┐
|
|
306
|
+
App2 (pool: 10) ──┼──→ PgBouncer (pool: 30) ──→ PostgreSQL
|
|
307
|
+
App3 (pool: 10) ──┘
|
|
308
|
+
```
|
|
309
|
+
|
|
310
|
+
Set application pool sizes low when using PgBouncer — the point is to reduce total connections, not add them up.
|