@1aboveio/app-dev-skills 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +30 -2
- package/package.json +34 -4
- package/skills/airflow/LICENSE +3 -0
- package/skills/airflow/SKILL.md +199 -0
- package/skills/airflow/release-check.mjs +93 -0
- package/skills/android-foundation/LICENSE +3 -0
- package/skills/android-foundation/SKILL.md +192 -0
- package/skills/android-foundation/assets/android-ci-job.yml +217 -0
- package/skills/android-foundation/references/ci.md +87 -0
- package/skills/android-foundation/references/gotchas.md +221 -0
- package/skills/android-foundation/references/setup.md +164 -0
- package/skills/android-foundation/release-check.mjs +93 -0
- package/skills/android-foundation/scripts/device-run.sh +176 -0
- package/skills/android-foundation/scripts/device-run.test.mjs +172 -0
- package/skills/app-debug/LICENSE +3 -0
- package/skills/app-debug/SKILL.md +110 -0
- package/skills/app-debug/references/common-errors.md +128 -0
- package/skills/app-debug/release-check.mjs +93 -0
- package/skills/better-auth/LICENSE +3 -0
- package/skills/better-auth/SKILL.md +598 -0
- package/skills/better-auth/references/feishu-api.md +270 -0
- package/skills/better-auth/release-check.mjs +93 -0
- package/skills/data-catalog/LICENSE +3 -0
- package/skills/data-catalog/SKILL.md +207 -0
- package/skills/data-catalog/agents/openai.yaml +3 -0
- package/skills/data-catalog/references/data-catalog-conventions.md +76 -0
- package/skills/data-catalog/references/field-definitions.md +123 -0
- package/skills/data-catalog/references/format.md +130 -0
- package/skills/data-catalog/release-check.mjs +93 -0
- package/skills/data-catalog/scripts/data-catalog.test.mjs +36 -0
- package/skills/data-catalog/scripts/data_catalog.py +1435 -0
- package/skills/data-catalog/scripts/persist-conventions-core.mjs +117 -0
- package/skills/data-catalog/scripts/persist-conventions.mjs +17 -0
- package/skills/data-catalog/scripts/test_data_catalog.py +603 -0
- package/skills/ios-foundation/LICENSE +3 -0
- package/skills/ios-foundation/SKILL.md +184 -0
- package/skills/ios-foundation/references/setup.md +243 -0
- package/skills/ios-foundation/release-check.mjs +93 -0
- package/skills/monorepo/LICENSE +3 -0
- package/skills/monorepo/SKILL.md +462 -0
- package/skills/monorepo/release-check.mjs +93 -0
- package/skills/nextjs-fullstack/LICENSE +3 -0
- package/skills/nextjs-fullstack/SKILL.md +211 -0
- package/skills/nextjs-fullstack/conventions.md +318 -0
- package/skills/nextjs-fullstack/frontend-conventions.md +61 -0
- package/skills/nextjs-fullstack/nextjs16.md +287 -0
- package/skills/nextjs-fullstack/release-check.mjs +93 -0
- package/skills/nextjs-fullstack/server-actions.md +409 -0
- package/skills/prisma-setup/LICENSE +3 -0
- package/skills/prisma-setup/SKILL.md +181 -0
- package/skills/prisma-setup/nextjs.md +258 -0
- package/skills/prisma-setup/release-check.mjs +93 -0
- package/skills/prisma-setup/turborepo.md +301 -0
- package/skills/pyspark/LICENSE +3 -0
- package/skills/pyspark/SKILL.md +127 -0
- package/skills/pyspark/assets/templates/etl.py +748 -0
- package/skills/pyspark/assets/templates/utils/__init__.py +1 -0
- package/skills/pyspark/assets/templates/utils/hudi_metadata.py +14 -0
- package/skills/pyspark/references/diagnosis-and-profiling.md +169 -0
- package/skills/pyspark/references/etl-contract.md +129 -0
- package/skills/pyspark/references/production-validation.md +131 -0
- package/skills/pyspark/references/reconciliation.md +38 -0
- package/skills/pyspark/references/transformation-design.md +190 -0
- package/skills/pyspark/references/velocity-feature-calculation.md +193 -0
- package/skills/pyspark/release-check.mjs +93 -0
- package/skills/pyspark/scripts/spark_eventlog_summary.py +223 -0
- package/skills/python-backend/LICENSE +3 -0
- package/skills/python-backend/SKILL.md +325 -0
- package/skills/python-backend/assets/uv-install.sh +48 -0
- package/skills/python-backend/references/uv-installs.md +83 -0
- package/skills/python-backend/release-check.mjs +93 -0
- package/skills/shadcn/LICENSE +3 -0
- package/skills/shadcn/SKILL.md +120 -0
- package/skills/shadcn/assets/shadcn-small.png +0 -0
- package/skills/shadcn/assets/shadcn.png +0 -0
- package/skills/shadcn/cli.md +411 -0
- package/skills/shadcn/customization.md +224 -0
- package/skills/shadcn/mcp.md +101 -0
- package/skills/shadcn/release-check.mjs +93 -0
- package/skills/shadcn/rules/base-vs-radix.md +323 -0
- package/skills/shadcn/rules/component-selection.md +67 -0
- package/skills/shadcn/rules/composition.md +195 -0
- package/skills/shadcn/rules/data-table.md +201 -0
- package/skills/shadcn/rules/forms.md +255 -0
- package/skills/shadcn/rules/icons.md +103 -0
- package/skills/shadcn/rules/styling.md +167 -0
- package/skills/zod-v4/LICENSE +3 -0
- package/skills/zod-v4/SKILL.md +286 -0
- package/skills/zod-v4/release-check.mjs +93 -0
|
@@ -0,0 +1,193 @@
|
|
|
1
|
+
# Velocity Feature Calculation Best Practices
|
|
2
|
+
|
|
3
|
+
Use these practices for rolling counts and sums, distinct counterparties,
|
|
4
|
+
temporal velocity features, and FIFO/path calculations over event histories.
|
|
5
|
+
The snippets illustrate the design; adapt schemas and semantics to the job's
|
|
6
|
+
required behavior.
|
|
7
|
+
|
|
8
|
+
## Use Built-Ins When Velocity Metrics Are Independent
|
|
9
|
+
|
|
10
|
+
For ordinary rolling counts and sums where same-time peers share one frame,
|
|
11
|
+
prefer a Spark range window:
|
|
12
|
+
|
|
13
|
+
```python
|
|
14
|
+
from pyspark.sql import Window, functions as F
|
|
15
|
+
|
|
16
|
+
last_24h = (
|
|
17
|
+
Window.partitionBy("entity_id")
|
|
18
|
+
.orderBy(F.col("event_time").cast("long"))
|
|
19
|
+
.rangeBetween(-24 * 60 * 60, 0)
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
velocity = (
|
|
23
|
+
events.withColumn("event_count_24h", F.count(F.lit(1)).over(last_24h))
|
|
24
|
+
.withColumn("amount_usd_24h", F.sum("amount_usd").over(last_24h))
|
|
25
|
+
)
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
This stays inside Catalyst and avoids Python serialization. It is not sufficient
|
|
29
|
+
when same-time visibility uses secondary keys, when distinct state requires
|
|
30
|
+
reference-counted eviction, or when velocity and FIFO transitions must share
|
|
31
|
+
one exact sequence.
|
|
32
|
+
|
|
33
|
+
## Keep Coupled State In One Pure Sweep
|
|
34
|
+
|
|
35
|
+
When rolling windows and FIFO/path transitions share one exact sequence,
|
|
36
|
+
isolate that sequence as a pure algorithm before choosing how Spark executes
|
|
37
|
+
it:
|
|
38
|
+
|
|
39
|
+
```python
|
|
40
|
+
from datetime import timedelta
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def sweep_events(events):
|
|
44
|
+
ordered = sorted(
|
|
45
|
+
events,
|
|
46
|
+
key=lambda event: (
|
|
47
|
+
event.event_time,
|
|
48
|
+
event.transaction_id,
|
|
49
|
+
event.event_id,
|
|
50
|
+
),
|
|
51
|
+
)
|
|
52
|
+
windows = {
|
|
53
|
+
"1h": RollingWindow(timedelta(hours=1)),
|
|
54
|
+
"24h": RollingWindow(timedelta(hours=24)),
|
|
55
|
+
}
|
|
56
|
+
fifo = FifoState()
|
|
57
|
+
|
|
58
|
+
for event in ordered:
|
|
59
|
+
for window in windows.values():
|
|
60
|
+
window.append(event)
|
|
61
|
+
path = fifo.advance(event)
|
|
62
|
+
yield build_result(event, windows, path)
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
This is the semantic core, not a recommendation to wrap it in `F.udf`. Select
|
|
66
|
+
the Spark execution API using the decision table in
|
|
67
|
+
[Transformation Design](transformation-design.md): built-in windows first,
|
|
68
|
+
then bounded higher-order aggregation, grouped pandas only for bounded groups,
|
|
69
|
+
and a different engine/design for large sequential state.
|
|
70
|
+
|
|
71
|
+
Whichever bridge is chosen, emit one result per unique output identity and
|
|
72
|
+
measure it against the pair relation or repeated grouping it replaces.
|
|
73
|
+
|
|
74
|
+
## Make Window Boundaries Literal
|
|
75
|
+
|
|
76
|
+
For a current-inclusive horizon `H`, evict only events strictly older than the
|
|
77
|
+
cutoff:
|
|
78
|
+
|
|
79
|
+
```python
|
|
80
|
+
from collections import Counter, deque
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
class RollingWindow:
|
|
84
|
+
def __init__(self, horizon):
|
|
85
|
+
self.horizon = horizon
|
|
86
|
+
self.events = deque()
|
|
87
|
+
self.count = 0
|
|
88
|
+
self.counterparties = Counter()
|
|
89
|
+
|
|
90
|
+
def append(self, event):
|
|
91
|
+
cutoff = event.event_time - self.horizon
|
|
92
|
+
while self.events and self.events[0].event_time < cutoff:
|
|
93
|
+
self.remove(self.events.popleft())
|
|
94
|
+
|
|
95
|
+
self.events.append(event) # current event is visible
|
|
96
|
+
self.count += 1
|
|
97
|
+
if event.counterparty_key is not None:
|
|
98
|
+
self.counterparties[event.counterparty_key] += 1
|
|
99
|
+
|
|
100
|
+
def remove(self, event):
|
|
101
|
+
self.count -= 1
|
|
102
|
+
key = event.counterparty_key
|
|
103
|
+
if key is not None:
|
|
104
|
+
self.counterparties[key] -= 1
|
|
105
|
+
if self.counterparties[key] == 0:
|
|
106
|
+
del self.counterparties[key]
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
Using `<= cutoff` makes the lower boundary exclusive. Test the exact boundary
|
|
110
|
+
and one smallest supported time unit beyond it.
|
|
111
|
+
|
|
112
|
+
A reference-counted map is required for distinct counterparties because a set
|
|
113
|
+
cannot represent repeated counterparties during eviction.
|
|
114
|
+
|
|
115
|
+
## Define Same-Timestamp Visibility
|
|
116
|
+
|
|
117
|
+
Use the complete stable order, for example:
|
|
118
|
+
|
|
119
|
+
```text
|
|
120
|
+
(event_time, transaction_id, event_id)
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
If same-time events see earlier tie-ordered events, update and emit in that
|
|
124
|
+
order. If they should all see the same batch, emit the timestamp batch before
|
|
125
|
+
updating state. Never let incidental Spark input order define visibility.
|
|
126
|
+
|
|
127
|
+
## Use Numerically Stable Rolling State
|
|
128
|
+
|
|
129
|
+
Repeated floating addition and eviction can leave residue. Use compensated
|
|
130
|
+
addition/removal, or exact decimal state when the domain requires exact money:
|
|
131
|
+
|
|
132
|
+
```python
|
|
133
|
+
class CompensatedSum:
|
|
134
|
+
def __init__(self):
|
|
135
|
+
self.total = 0.0
|
|
136
|
+
self.correction = 0.0
|
|
137
|
+
|
|
138
|
+
def add(self, value):
|
|
139
|
+
updated = self.total + value
|
|
140
|
+
if abs(self.total) >= abs(value):
|
|
141
|
+
self.correction += (self.total - updated) + value
|
|
142
|
+
else:
|
|
143
|
+
self.correction += (value - updated) + self.total
|
|
144
|
+
self.total = updated
|
|
145
|
+
|
|
146
|
+
def remove(self, value):
|
|
147
|
+
self.add(-value)
|
|
148
|
+
|
|
149
|
+
def value(self):
|
|
150
|
+
return self.total + self.correction
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
Test values with different magnitudes, evict them, and assert that the previous
|
|
154
|
+
state is restored.
|
|
155
|
+
|
|
156
|
+
## Combine Related State
|
|
157
|
+
|
|
158
|
+
Within the same sweep:
|
|
159
|
+
|
|
160
|
+
1. Evict expired events from each horizon.
|
|
161
|
+
2. Add the current event according to visibility semantics.
|
|
162
|
+
3. Emit counts, sums, missing-value counts, directional metrics, and distinct
|
|
163
|
+
counterparties.
|
|
164
|
+
4. Advance FIFO/path state using the same stable order.
|
|
165
|
+
5. Emit one result keyed by output identity.
|
|
166
|
+
|
|
167
|
+
This removes the pair relation and avoids grouping and sorting the same history
|
|
168
|
+
for each feature family.
|
|
169
|
+
|
|
170
|
+
## Prove Parity And Plan Shape
|
|
171
|
+
|
|
172
|
+
Keep the old self-join only as a bounded test oracle. Cover exact 1h/24h
|
|
173
|
+
boundaries, one unit outside, same-time ties, nulls, both directions,
|
|
174
|
+
missing/present FX, repeated counterparties, path transitions, and floating
|
|
175
|
+
residue.
|
|
176
|
+
|
|
177
|
+
Compare full rows in both directions, add literal boundary assertions, and
|
|
178
|
+
assert the candidate physical plan has no customer/time interval join.
|
|
179
|
+
|
|
180
|
+
## Measure Residual Per-Key Memory
|
|
181
|
+
|
|
182
|
+
`groupBy + collect_list` changes quadratic distributed work into linear per-key
|
|
183
|
+
work, but one task still holds the entire largest key. Measure and report
|
|
184
|
+
`max_rows_per_key`, then production-canary the largest known shape. Use bounded
|
|
185
|
+
state or exact temporal bucketing if one key no longer fits.
|
|
186
|
+
|
|
187
|
+
## Avoid These False Fixes
|
|
188
|
+
|
|
189
|
+
- Do not rely on repartitioning to split one hot grouping key.
|
|
190
|
+
- Do not drop history, events, or feature formulas to gain speed.
|
|
191
|
+
- Do not present Arrow transport, broadcasting, or explicit persistence as the
|
|
192
|
+
performance fix; reduce the logical work instead.
|
|
193
|
+
- Do not report validate-only runtime as end-to-end write performance.
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { spawnSync } from 'node:child_process';
|
|
3
|
+
import { mkdirSync, readFileSync, writeFileSync } from 'node:fs';
|
|
4
|
+
import { homedir } from 'node:os';
|
|
5
|
+
import { dirname, join } from 'node:path';
|
|
6
|
+
import { fileURLToPath } from 'node:url';
|
|
7
|
+
|
|
8
|
+
const DAY_MS = 24 * 60 * 60 * 1000;
|
|
9
|
+
const WEEK_MS = 7 * DAY_MS;
|
|
10
|
+
const here = dirname(fileURLToPath(import.meta.url));
|
|
11
|
+
|
|
12
|
+
function readJson(path, fallback = {}) {
|
|
13
|
+
try { return JSON.parse(readFileSync(path, 'utf8')); } catch { return fallback; }
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
function frontmatterValue(text, key) {
|
|
17
|
+
const frontmatter = /^---\n([\s\S]*?)\n---/.exec(text)?.[1] ?? '';
|
|
18
|
+
const match = new RegExp('^' + key + ':\\s*([^\\n]+)$', 'm').exec(frontmatter);
|
|
19
|
+
return match?.[1].trim().replace(/^['"]|['"]$/g, '') ?? null;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
function repositoryOf(source) {
|
|
23
|
+
const value = String(source ?? '').trim();
|
|
24
|
+
const match = /^(?:git@github\.com:|https:\/\/github\.com\/)?([A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+?)(?:\.git)?$/.exec(value);
|
|
25
|
+
return match?.[1] ?? null;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
function stableVersion(value) {
|
|
29
|
+
const match = /^v?(\d+)\.(\d+)\.(\d+)$/.exec(String(value ?? '').trim());
|
|
30
|
+
return match ? match.slice(1).map(Number) : null;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
function newer(left, right) {
|
|
34
|
+
for (let index = 0; index < 3; index += 1) {
|
|
35
|
+
if (left[index] !== right[index]) return left[index] > right[index];
|
|
36
|
+
}
|
|
37
|
+
return false;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
function writeCache(path, value) {
|
|
41
|
+
try {
|
|
42
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
43
|
+
writeFileSync(path, JSON.stringify(value));
|
|
44
|
+
} catch { /* A read-only cache must not block skill use. */ }
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
function main(now = Date.now()) {
|
|
48
|
+
if (process.env.ONEABOVEIO_NO_UPDATE_CHECK) return;
|
|
49
|
+
const skill = readFileSync(join(here, 'SKILL.md'), 'utf8');
|
|
50
|
+
const name = frontmatterValue(skill, 'name');
|
|
51
|
+
const localVersion = stableVersion(frontmatterValue(skill, 'release-version'));
|
|
52
|
+
if (!name || !localVersion) return;
|
|
53
|
+
|
|
54
|
+
const lock = readJson(join(process.env.HOME || homedir(), '.agents', '.skill-lock.json'));
|
|
55
|
+
const entry = lock.skills?.[name];
|
|
56
|
+
const repository = repositoryOf(entry?.sourceUrl || entry?.source);
|
|
57
|
+
if (!repository) return;
|
|
58
|
+
|
|
59
|
+
const cacheRoot = process.env.XDG_CACHE_HOME || join(process.env.HOME || homedir(), '.cache');
|
|
60
|
+
const cachePath = join(cacheRoot, '1aboveio-skills', 'release-check.json');
|
|
61
|
+
const all = readJson(cachePath);
|
|
62
|
+
const cache = all[repository] && typeof all[repository] === 'object' ? all[repository] : {};
|
|
63
|
+
|
|
64
|
+
if (!cache.lastCheck || now - cache.lastCheck >= DAY_MS) {
|
|
65
|
+
const result = spawnSync('gh', ['api', 'repos/' + repository + '/releases/latest', '--jq', '.tag_name'], {
|
|
66
|
+
encoding: 'utf8',
|
|
67
|
+
stdio: ['ignore', 'pipe', 'ignore'],
|
|
68
|
+
timeout: 1500,
|
|
69
|
+
});
|
|
70
|
+
cache.lastCheck = now;
|
|
71
|
+
if (result.status === 0 && stableVersion(result.stdout)) cache.latestTag = result.stdout.trim();
|
|
72
|
+
else delete cache.latestTag;
|
|
73
|
+
all[repository] = cache;
|
|
74
|
+
writeCache(cachePath, all);
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
const latestVersion = stableVersion(cache.latestTag);
|
|
78
|
+
if (!latestVersion || !newer(latestVersion, localVersion)) return;
|
|
79
|
+
if (cache.notifiedTag === cache.latestTag && cache.notifiedAt && now - cache.notifiedAt < WEEK_MS) return;
|
|
80
|
+
|
|
81
|
+
cache.notifiedTag = cache.latestTag;
|
|
82
|
+
cache.notifiedAt = now;
|
|
83
|
+
all[repository] = cache;
|
|
84
|
+
writeCache(cachePath, all);
|
|
85
|
+
const command = `npx skills@1.5.23 add '${repository}#${cache.latestTag}' --global --agent universal claude-code --skill ${name} --yes`;
|
|
86
|
+
process.stdout.write(
|
|
87
|
+
`UPDATE_AVAILABLE: A newer 1AboveIO skills release (${cache.latestTag}) is available; this skill is `
|
|
88
|
+
+ `v${localVersion.join('.')}. Ask once whether to refresh ${name}. On approval run \`${command}\`. `
|
|
89
|
+
+ 'The refreshed instructions apply next session. Either way, continue the current task and do not ask again this session.\n',
|
|
90
|
+
);
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
try { main(); } catch { /* Update checks are advisory and always fail silent. */ }
|
|
@@ -0,0 +1,223 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Summarize stage attempts and final SQL plans from a Spark JSON event log."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import argparse
|
|
7
|
+
import json
|
|
8
|
+
import re
|
|
9
|
+
import sys
|
|
10
|
+
from collections.abc import Iterable
|
|
11
|
+
from typing import Any
|
|
12
|
+
|
|
13
|
+
METRICS = (
|
|
14
|
+
"input_bytes",
|
|
15
|
+
"input_records",
|
|
16
|
+
"output_bytes",
|
|
17
|
+
"output_records",
|
|
18
|
+
"shuffle_read_bytes",
|
|
19
|
+
"shuffle_read_records",
|
|
20
|
+
"shuffle_write_bytes",
|
|
21
|
+
"shuffle_write_records",
|
|
22
|
+
"memory_spill_bytes",
|
|
23
|
+
"disk_spill_bytes",
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _patterns(values: Iterable[str]) -> dict[str, re.Pattern[str]]:
|
|
28
|
+
result: dict[str, re.Pattern[str]] = {}
|
|
29
|
+
for value in values:
|
|
30
|
+
name, separator, pattern = value.partition("=")
|
|
31
|
+
if not separator or not name or not pattern:
|
|
32
|
+
raise ValueError(
|
|
33
|
+
f"invalid --plan-pattern {value!r}; expected NAME=REGEX"
|
|
34
|
+
)
|
|
35
|
+
result[name] = re.compile(pattern, re.IGNORECASE)
|
|
36
|
+
return result
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _add(stage: dict[str, Any], name: str, value: Any) -> None:
|
|
40
|
+
stage[name] = stage.get(name, 0) + (value or 0)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def summarize(
|
|
44
|
+
lines: Iterable[str], plan_patterns: dict[str, re.Pattern[str]]
|
|
45
|
+
) -> dict[str, Any]:
|
|
46
|
+
app: dict[str, Any] = {}
|
|
47
|
+
stages: dict[tuple[int, int], dict[str, Any]] = {}
|
|
48
|
+
sql_execution_ids: set[Any] = set()
|
|
49
|
+
sql_plans: dict[Any, str] = {}
|
|
50
|
+
anonymous_sql_id = 0
|
|
51
|
+
|
|
52
|
+
for raw in lines:
|
|
53
|
+
try:
|
|
54
|
+
event = json.loads(raw)
|
|
55
|
+
except (json.JSONDecodeError, TypeError):
|
|
56
|
+
continue
|
|
57
|
+
|
|
58
|
+
kind = event.get("Event")
|
|
59
|
+
if kind == "SparkListenerApplicationStart":
|
|
60
|
+
app.update(
|
|
61
|
+
{
|
|
62
|
+
"app_id": event.get("App ID"),
|
|
63
|
+
"app_name": event.get("App Name"),
|
|
64
|
+
"start_time_ms": event.get("Timestamp"),
|
|
65
|
+
}
|
|
66
|
+
)
|
|
67
|
+
elif kind == "SparkListenerApplicationEnd":
|
|
68
|
+
app["end_time_ms"] = event.get("Timestamp")
|
|
69
|
+
elif kind in {
|
|
70
|
+
"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart",
|
|
71
|
+
"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate",
|
|
72
|
+
}:
|
|
73
|
+
execution_id = event.get("executionId")
|
|
74
|
+
if execution_id is None:
|
|
75
|
+
execution_id = f"anonymous-{anonymous_sql_id}"
|
|
76
|
+
anonymous_sql_id += 1
|
|
77
|
+
sql_execution_ids.add(execution_id)
|
|
78
|
+
sql_plans[execution_id] = event.get("physicalPlanDescription") or ""
|
|
79
|
+
elif kind == "SparkListenerStageCompleted":
|
|
80
|
+
info = event.get("Stage Info") or {}
|
|
81
|
+
stage_id = info.get("Stage ID")
|
|
82
|
+
if stage_id is None:
|
|
83
|
+
continue
|
|
84
|
+
attempt_id = info.get("Stage Attempt ID", 0)
|
|
85
|
+
key = (stage_id, attempt_id)
|
|
86
|
+
stage = stages.setdefault(
|
|
87
|
+
key,
|
|
88
|
+
{"stage_id": stage_id, "stage_attempt_id": attempt_id},
|
|
89
|
+
)
|
|
90
|
+
stage.update(
|
|
91
|
+
{
|
|
92
|
+
"name": info.get("Stage Name"),
|
|
93
|
+
"task_count": info.get("Number of Tasks", 0),
|
|
94
|
+
"submission_time_ms": info.get("Submission Time"),
|
|
95
|
+
"completion_time_ms": info.get("Completion Time"),
|
|
96
|
+
}
|
|
97
|
+
)
|
|
98
|
+
if (
|
|
99
|
+
stage["submission_time_ms"] is not None
|
|
100
|
+
and stage["completion_time_ms"] is not None
|
|
101
|
+
):
|
|
102
|
+
stage["wall_ms"] = (
|
|
103
|
+
stage["completion_time_ms"] - stage["submission_time_ms"]
|
|
104
|
+
)
|
|
105
|
+
elif kind == "SparkListenerTaskEnd":
|
|
106
|
+
stage_id = event.get("Stage ID")
|
|
107
|
+
if stage_id is None:
|
|
108
|
+
continue
|
|
109
|
+
attempt_id = event.get("Stage Attempt ID", 0)
|
|
110
|
+
key = (stage_id, attempt_id)
|
|
111
|
+
stage = stages.setdefault(
|
|
112
|
+
key,
|
|
113
|
+
{"stage_id": stage_id, "stage_attempt_id": attempt_id},
|
|
114
|
+
)
|
|
115
|
+
info = event.get("Task Info") or {}
|
|
116
|
+
metrics = event.get("Task Metrics") or {}
|
|
117
|
+
launch = info.get("Launch Time")
|
|
118
|
+
finish = info.get("Finish Time")
|
|
119
|
+
duration = finish - launch if launch is not None and finish is not None else 0
|
|
120
|
+
stage["max_task_ms"] = max(stage.get("max_task_ms", 0), duration)
|
|
121
|
+
_add(stage, "task_end_count", 1)
|
|
122
|
+
_add(stage, "memory_spill_bytes", metrics.get("Memory Bytes Spilled"))
|
|
123
|
+
_add(stage, "disk_spill_bytes", metrics.get("Disk Bytes Spilled"))
|
|
124
|
+
|
|
125
|
+
input_metrics = metrics.get("Input Metrics") or {}
|
|
126
|
+
output_metrics = metrics.get("Output Metrics") or {}
|
|
127
|
+
shuffle_read = metrics.get("Shuffle Read Metrics") or {}
|
|
128
|
+
shuffle_write = metrics.get("Shuffle Write Metrics") or {}
|
|
129
|
+
_add(stage, "input_bytes", input_metrics.get("Bytes Read"))
|
|
130
|
+
_add(stage, "input_records", input_metrics.get("Records Read"))
|
|
131
|
+
_add(stage, "output_bytes", output_metrics.get("Bytes Written"))
|
|
132
|
+
_add(stage, "output_records", output_metrics.get("Records Written"))
|
|
133
|
+
_add(
|
|
134
|
+
stage,
|
|
135
|
+
"shuffle_read_bytes",
|
|
136
|
+
(shuffle_read.get("Remote Bytes Read") or 0)
|
|
137
|
+
+ (shuffle_read.get("Local Bytes Read") or 0),
|
|
138
|
+
)
|
|
139
|
+
_add(
|
|
140
|
+
stage,
|
|
141
|
+
"shuffle_read_records",
|
|
142
|
+
shuffle_read.get("Total Records Read"),
|
|
143
|
+
)
|
|
144
|
+
_add(
|
|
145
|
+
stage,
|
|
146
|
+
"shuffle_write_bytes",
|
|
147
|
+
shuffle_write.get("Shuffle Bytes Written"),
|
|
148
|
+
)
|
|
149
|
+
_add(
|
|
150
|
+
stage,
|
|
151
|
+
"shuffle_write_records",
|
|
152
|
+
shuffle_write.get("Shuffle Records Written"),
|
|
153
|
+
)
|
|
154
|
+
|
|
155
|
+
completed = sorted(
|
|
156
|
+
(stage for stage in stages.values() if "wall_ms" in stage),
|
|
157
|
+
key=lambda stage: (stage["stage_id"], stage["stage_attempt_id"]),
|
|
158
|
+
)
|
|
159
|
+
totals = {
|
|
160
|
+
name: sum(stage.get(name, 0) for stage in completed) for name in METRICS
|
|
161
|
+
}
|
|
162
|
+
sql = {
|
|
163
|
+
"execution_count": len(sql_execution_ids),
|
|
164
|
+
"plan_pattern_counts": {
|
|
165
|
+
name: sum(len(pattern.findall(plan)) for plan in sql_plans.values())
|
|
166
|
+
for name, pattern in plan_patterns.items()
|
|
167
|
+
},
|
|
168
|
+
}
|
|
169
|
+
if app.get("start_time_ms") is not None and app.get("end_time_ms") is not None:
|
|
170
|
+
app["eventlog_wall_ms"] = app["end_time_ms"] - app["start_time_ms"]
|
|
171
|
+
|
|
172
|
+
def largest(metric: str) -> dict[str, Any]:
|
|
173
|
+
return max(completed, key=lambda stage: stage.get(metric, 0), default={})
|
|
174
|
+
|
|
175
|
+
return {
|
|
176
|
+
"application": app,
|
|
177
|
+
"sql": sql,
|
|
178
|
+
"completed_stage_count": len({stage["stage_id"] for stage in completed}),
|
|
179
|
+
"completed_stage_attempt_count": len(completed),
|
|
180
|
+
"totals": totals,
|
|
181
|
+
"dominant_stage": largest("wall_ms"),
|
|
182
|
+
"max_task_stage": largest("max_task_ms"),
|
|
183
|
+
"max_shuffle_write_records_stage": largest("shuffle_write_records"),
|
|
184
|
+
"stages": completed,
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def main() -> int:
|
|
189
|
+
parser = argparse.ArgumentParser(description=__doc__)
|
|
190
|
+
parser.add_argument(
|
|
191
|
+
"--plan-pattern",
|
|
192
|
+
action="append",
|
|
193
|
+
default=[],
|
|
194
|
+
metavar="NAME=REGEX",
|
|
195
|
+
help="count matching physical-plan fragments; may be repeated",
|
|
196
|
+
)
|
|
197
|
+
parser.add_argument(
|
|
198
|
+
"--include-stages",
|
|
199
|
+
action="store_true",
|
|
200
|
+
help="include every completed stage instead of only aggregate/top stages",
|
|
201
|
+
)
|
|
202
|
+
args = parser.parse_args()
|
|
203
|
+
|
|
204
|
+
try:
|
|
205
|
+
patterns = _patterns(args.plan_pattern)
|
|
206
|
+
except (ValueError, re.error) as error:
|
|
207
|
+
parser.error(str(error))
|
|
208
|
+
|
|
209
|
+
result = summarize(sys.stdin, patterns)
|
|
210
|
+
if not result["application"].get("app_id"):
|
|
211
|
+
parser.error(
|
|
212
|
+
"no SparkListenerApplicationStart record found; verify the event-log "
|
|
213
|
+
"path and use shell pipefail so upstream read failures remain visible"
|
|
214
|
+
)
|
|
215
|
+
if not args.include_stages:
|
|
216
|
+
result.pop("stages", None)
|
|
217
|
+
json.dump(result, sys.stdout, indent=2, sort_keys=True)
|
|
218
|
+
sys.stdout.write("\n")
|
|
219
|
+
return 0
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
if __name__ == "__main__":
|
|
223
|
+
raise SystemExit(main())
|