@mmerterden/multi-agent-pipeline 16.25.0 → 16.27.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +77 -0
- package/README.md +1 -1
- package/README.tr.md +1 -1
- package/install/templates/claude-hooks.json +32 -1
- package/package.json +1 -1
- package/pipeline/commands/multi-agent/help/SKILL.md +2 -0
- package/pipeline/commands/multi-agent/refactor/SKILL.md +23 -1
- package/pipeline/commands/multi-agent/search/SKILL.md +28 -0
- package/pipeline/commands/multi-agent/setup/SKILL.md +18 -44
- package/pipeline/commands/multi-agent/status/SKILL.md +9 -0
- package/pipeline/lib/credential-inventory.sh +142 -18
- package/pipeline/lib/fetch-crashlytics.sh +123 -28
- package/pipeline/multi-agent-refs/features/url-enrichment.md +1 -1
- package/pipeline/multi-agent-refs/features/visual-evidence.md +5 -0
- package/pipeline/multi-agent-refs/keychain.md +65 -20
- package/pipeline/multi-agent-refs/knowledge.md +27 -0
- package/pipeline/multi-agent-refs/phases/operations.md +7 -1
- package/pipeline/multi-agent-refs/phases/phase-1-analysis.md +3 -1
- package/pipeline/multi-agent-refs/phases/phase-4-review.md +1 -8
- package/pipeline/multi-agent-refs/phases/phase-7-report.md +11 -21
- package/pipeline/multi-agent-refs/picker-contract.md +1 -1
- package/pipeline/multi-agent-refs/refactor/observations.md +81 -0
- package/pipeline/multi-agent-refs/setup/firebase.md +151 -0
- package/pipeline/schemas/learnings-ledger.schema.json +5 -0
- package/pipeline/schemas/prefs.schema.json +31 -3
- package/pipeline/schemas/skill-observation.schema.json +73 -0
- package/pipeline/scripts/capture-flush.sh +158 -0
- package/pipeline/scripts/capture-resume.sh +87 -0
- package/pipeline/scripts/crush-json.mjs +283 -0
- package/pipeline/scripts/firebase-app-discovery.sh +114 -0
- package/pipeline/scripts/keychain-save.sh +5 -8
- package/pipeline/scripts/keychain.py +76 -14
- package/pipeline/scripts/learn-from-transcripts.mjs +625 -0
- package/pipeline/scripts/learning-curve.mjs +22 -4
- package/pipeline/scripts/learnings-ledger.mjs +86 -12
- package/pipeline/scripts/note-session.sh +187 -0
- package/pipeline/scripts/observations.mjs +347 -0
- package/pipeline/scripts/offload-ref.sh +45 -2
- package/pipeline/scripts/pre-commit-check.sh +31 -1
- package/pipeline/scripts/scan-agent-config.sh +12 -3
- package/pipeline/scripts/skill-siblings.mjs +187 -0
- package/pipeline/scripts/triage-memory.mjs +73 -9
- package/pipeline/skills/.skill-manifest.json +1 -1
- package/pipeline/skills/shared/core/multi-agent-refactor/SKILL.md +23 -1
- package/pipeline/skills/shared/core/multi-agent-search/SKILL.md +28 -0
- package/pipeline/skills/shared/core/multi-agent-setup/SKILL.md +37 -6
- package/pipeline/skills/shared/core/multi-agent-status/SKILL.md +9 -0
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
#
|
|
3
|
+
# capture-flush.sh - write what a run has already learned into the durable
|
|
4
|
+
# stores, without needing the run to reach its end.
|
|
5
|
+
#
|
|
6
|
+
# WHY THIS EXISTS
|
|
7
|
+
#
|
|
8
|
+
# Every persistent write used to live in Phase 7: triage ingest, the learnings
|
|
9
|
+
# ledger distill, the knowledge-base append, the code-graph refresh. And Phase 7
|
|
10
|
+
# is, by the pipeline's own admission in features/code-graph.md, the phase a run
|
|
11
|
+
# is LEAST likely to reach. A run killed in Phase 3, a session that hits its
|
|
12
|
+
# context ceiling, a crash after review - each one threw away everything it had
|
|
13
|
+
# established, and the next run on the same repo rediscovered it from scratch.
|
|
14
|
+
#
|
|
15
|
+
# So the writes move here, and Phase 7 becomes the LAST flush rather than the
|
|
16
|
+
# only one. Phase boundaries call this, and so does SessionEnd. Nothing about
|
|
17
|
+
# the trigger depends on a model noticing that a moment qualifies: it hangs on
|
|
18
|
+
# a phase transition and on process exit, both objectively visible without any
|
|
19
|
+
# agent's cooperation.
|
|
20
|
+
#
|
|
21
|
+
# What it does NOT do: call a model. Everything here is derived from artefacts
|
|
22
|
+
# already on disk (triage-output.json) plus agent-state.json. The parts of
|
|
23
|
+
# Phase 7 that genuinely need a model - the knowledge-base extraction, the
|
|
24
|
+
# per-repo memory synthesis - stay in Phase 7, because a hook cannot think.
|
|
25
|
+
#
|
|
26
|
+
# Usage:
|
|
27
|
+
# ./capture-flush.sh [--state <agent-state.json>] [--if-stale] [--json] [--quiet]
|
|
28
|
+
#
|
|
29
|
+
# --state the run to flush. Default: resolved from the newest task dir
|
|
30
|
+
# under $HOME/.claude/logs/multi-agent (see resolve_state).
|
|
31
|
+
# --if-stale flush only when the run did NOT complete Phase 7 - the
|
|
32
|
+
# SessionEnd case. A finished run has already flushed.
|
|
33
|
+
# --json machine-readable result for a caller that wants to count rows.
|
|
34
|
+
# --quiet no stdout. Exit status still distinguishes the outcomes.
|
|
35
|
+
#
|
|
36
|
+
# Exit codes:
|
|
37
|
+
# 0 flushed, or nothing to flush (both are fine outcomes)
|
|
38
|
+
# 1 bad usage
|
|
39
|
+
#
|
|
40
|
+
# It never exits non-zero because a store rejected a row. This runs from a hook,
|
|
41
|
+
# and a hook that fails a session over a bookkeeping write is worse than the
|
|
42
|
+
# bookkeeping it protects. Every failure is reported and swallowed.
|
|
43
|
+
|
|
44
|
+
set -uo pipefail
|
|
45
|
+
|
|
46
|
+
STATE=""
|
|
47
|
+
IF_STALE=0
|
|
48
|
+
JSON=0
|
|
49
|
+
QUIET=0
|
|
50
|
+
|
|
51
|
+
while [ "$#" -gt 0 ]; do
|
|
52
|
+
case "$1" in
|
|
53
|
+
--state) STATE="${2:-}"; shift 2 || shift ;;
|
|
54
|
+
--if-stale) IF_STALE=1; shift ;;
|
|
55
|
+
--json) JSON=1; shift ;;
|
|
56
|
+
--quiet) QUIET=1; shift ;;
|
|
57
|
+
-h|--help)
|
|
58
|
+
echo "usage: $0 [--state <agent-state.json>] [--if-stale] [--json] [--quiet]" >&2
|
|
59
|
+
exit 1 ;;
|
|
60
|
+
*)
|
|
61
|
+
echo "ERR: unexpected arg $1" >&2; exit 1 ;;
|
|
62
|
+
esac
|
|
63
|
+
done
|
|
64
|
+
|
|
65
|
+
LOGS="$HOME/.claude/logs/multi-agent"
|
|
66
|
+
SCRIPTS="$HOME/.claude/scripts"
|
|
67
|
+
[ -d "$SCRIPTS" ] || SCRIPTS="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)"
|
|
68
|
+
|
|
69
|
+
say() { [ "$QUIET" -eq 1 ] && return 0; printf '%s\n' "$1"; }
|
|
70
|
+
|
|
71
|
+
# Resolve the run to flush.
|
|
72
|
+
#
|
|
73
|
+
# Newest by mtime of agent-state.json, not by directory name: task ids do not
|
|
74
|
+
# sort chronologically (PROJ-1002 can run before PROJ-1001 does) and a
|
|
75
|
+
# name-sorted pick would flush the wrong run.
|
|
76
|
+
resolve_state() {
|
|
77
|
+
[ -d "$LOGS" ] || return 0
|
|
78
|
+
find "$LOGS" -name agent-state.json -type f -maxdepth 4 -print0 2>/dev/null \
|
|
79
|
+
| xargs -0 ls -t 2>/dev/null | head -1
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
if [ -z "$STATE" ]; then
|
|
83
|
+
STATE="$(resolve_state)"
|
|
84
|
+
fi
|
|
85
|
+
|
|
86
|
+
if [ -z "$STATE" ] || [ ! -f "$STATE" ]; then
|
|
87
|
+
say "capture-flush: no run state found - nothing to flush"
|
|
88
|
+
[ "$JSON" -eq 1 ] && printf '{"status":"noop","reason":"no-state"}\n'
|
|
89
|
+
exit 0
|
|
90
|
+
fi
|
|
91
|
+
|
|
92
|
+
if ! command -v jq >/dev/null 2>&1; then
|
|
93
|
+
say "capture-flush: jq unavailable - skipped"
|
|
94
|
+
[ "$JSON" -eq 1 ] && printf '{"status":"skipped","reason":"no-jq"}\n'
|
|
95
|
+
exit 0
|
|
96
|
+
fi
|
|
97
|
+
|
|
98
|
+
TASK_ID=$(jq -r '.taskId // .jiraId // empty' "$STATE" 2>/dev/null)
|
|
99
|
+
ARTIFACTS=$(jq -r '.artifactsPath // empty' "$STATE" 2>/dev/null)
|
|
100
|
+
WORKTREE=$(jq -r '.worktreePath // empty' "$STATE" 2>/dev/null)
|
|
101
|
+
PHASE7=$(jq -r '[.phases[]? | select((.id // "") == "7") | .status] | first // ""' "$STATE" 2>/dev/null)
|
|
102
|
+
|
|
103
|
+
if [ "$IF_STALE" -eq 1 ] && [ "$PHASE7" = "completed" ]; then
|
|
104
|
+
say "capture-flush: ${TASK_ID:-run} already completed Phase 7 - nothing stale"
|
|
105
|
+
[ "$JSON" -eq 1 ] && printf '{"status":"noop","reason":"already-flushed","taskId":"%s"}\n' "$TASK_ID"
|
|
106
|
+
exit 0
|
|
107
|
+
fi
|
|
108
|
+
|
|
109
|
+
# The triage artefact is the only input either store needs, and it has two homes:
|
|
110
|
+
# Phase 6 removes the worktree once the PR is open, so the salvaged copy under
|
|
111
|
+
# artifactsPath is tried FIRST. Reading the worktree path first would degrade
|
|
112
|
+
# silently for exactly the runs this script exists to rescue.
|
|
113
|
+
TRIAGE=""
|
|
114
|
+
for cand in "${ARTIFACTS:+$ARTIFACTS/triage-output.json}" "${WORKTREE:+$WORKTREE/triage-output.json}"; do
|
|
115
|
+
[ -n "$cand" ] && [ -f "$cand" ] && { TRIAGE="$cand"; break; }
|
|
116
|
+
done
|
|
117
|
+
|
|
118
|
+
INGESTED="skipped"
|
|
119
|
+
DISTILLED="skipped"
|
|
120
|
+
|
|
121
|
+
# Both writes are idempotent by contract, which is what makes calling this at
|
|
122
|
+
# every phase boundary safe: the second call through writes 0 rows.
|
|
123
|
+
#
|
|
124
|
+
# And 0 rows is a SUCCESS here. Both tools exit 2 on "nothing new", which is the
|
|
125
|
+
# normal outcome of every flush after the first - so exit 2 is folded into ok
|
|
126
|
+
# rather than reported as failure. Treating it as failure would have made the
|
|
127
|
+
# steady state of this script look broken.
|
|
128
|
+
run_store() {
|
|
129
|
+
local label="$1"; shift
|
|
130
|
+
local rc=0
|
|
131
|
+
"$@" >/dev/null 2>&1 || rc=$?
|
|
132
|
+
case "$rc" in
|
|
133
|
+
0) printf 'ok' ;;
|
|
134
|
+
2) printf 'ok-nothing-new' ;;
|
|
135
|
+
*) printf 'failed(%s)' "$rc" ;;
|
|
136
|
+
esac
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
if [ -n "$TRIAGE" ]; then
|
|
140
|
+
INGESTED=$(run_store ingest \
|
|
141
|
+
node "$SCRIPTS/triage-memory.mjs" ingest --triage "$TRIAGE" --state "$STATE")
|
|
142
|
+
if [ -n "$TASK_ID" ]; then
|
|
143
|
+
DISTILLED=$(run_store distill \
|
|
144
|
+
node "$SCRIPTS/learnings-ledger.mjs" from-triage --triage "$TRIAGE" --task "$TASK_ID")
|
|
145
|
+
fi
|
|
146
|
+
fi
|
|
147
|
+
|
|
148
|
+
if [ "$JSON" -eq 1 ]; then
|
|
149
|
+
printf '{"status":"flushed","taskId":"%s","triage":"%s","ingest":"%s","distill":"%s"}\n' \
|
|
150
|
+
"$TASK_ID" "${TRIAGE:-none}" "$INGESTED" "$DISTILLED"
|
|
151
|
+
fi
|
|
152
|
+
|
|
153
|
+
if [ -z "$TRIAGE" ]; then
|
|
154
|
+
say "capture-flush: ${TASK_ID:-run} has no triage output yet - nothing to flush"
|
|
155
|
+
else
|
|
156
|
+
say "capture-flush: ${TASK_ID:-run} triage=$INGESTED ledger=$DISTILLED"
|
|
157
|
+
fi
|
|
158
|
+
exit 0
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
#
|
|
3
|
+
# capture-resume.sh - one line at session start when a run was left unfinished.
|
|
4
|
+
#
|
|
5
|
+
# Runs from the SessionStart hook. Its whole job is to say what is pending and
|
|
6
|
+
# then get out of the way, so it is bound by three rules:
|
|
7
|
+
#
|
|
8
|
+
# 1. It never blocks. Exit 0 always, whatever it finds.
|
|
9
|
+
# 2. It never calls a model, and it never reads a transcript.
|
|
10
|
+
# 3. It prints at most a few lines. A session banner that scrolls is a banner
|
|
11
|
+
# the user learns to skip, which costs more than it saves.
|
|
12
|
+
#
|
|
13
|
+
# It answers two questions:
|
|
14
|
+
# - Is there a pipeline run that stopped before Phase 7? Name it and the resume
|
|
15
|
+
# command, because that run's work is recoverable and its findings are not
|
|
16
|
+
# yet in the durable stores until it flushes.
|
|
17
|
+
# - Is the pipeline's own observation queue stale? (>= REVIEW_DAYS since the
|
|
18
|
+
# last review, with at least one open observation.) Offer one line. The
|
|
19
|
+
# user's own work is never made to wait on it.
|
|
20
|
+
#
|
|
21
|
+
# Usage: ./capture-resume.sh [--json]
|
|
22
|
+
|
|
23
|
+
set -uo pipefail
|
|
24
|
+
|
|
25
|
+
JSON=0
|
|
26
|
+
[ "${1:-}" = "--json" ] && JSON=1
|
|
27
|
+
|
|
28
|
+
LOGS="$HOME/.claude/logs/multi-agent"
|
|
29
|
+
PIPELINE_MEM="$HOME/.claude/memory/multi-agent/_pipeline"
|
|
30
|
+
REVIEW_DAYS=7
|
|
31
|
+
|
|
32
|
+
STALE_TASK=""
|
|
33
|
+
STALE_PHASE=""
|
|
34
|
+
|
|
35
|
+
if [ -d "$LOGS" ] && command -v jq >/dev/null 2>&1; then
|
|
36
|
+
# Newest by mtime, not by name: task ids do not sort chronologically.
|
|
37
|
+
NEWEST=$(find "$LOGS" -name agent-state.json -type f -maxdepth 4 -print0 2>/dev/null \
|
|
38
|
+
| xargs -0 ls -t 2>/dev/null | head -1)
|
|
39
|
+
if [ -n "$NEWEST" ] && [ -f "$NEWEST" ]; then
|
|
40
|
+
P7=$(jq -r '[.phases[]? | select((.id // "") == "7") | .status] | first // ""' "$NEWEST" 2>/dev/null)
|
|
41
|
+
STATUS=$(jq -r '.status // ""' "$NEWEST" 2>/dev/null)
|
|
42
|
+
if [ "$P7" != "completed" ] && [ "$STATUS" != "completed" ]; then
|
|
43
|
+
STALE_TASK=$(jq -r '.taskId // .jiraId // ""' "$NEWEST" 2>/dev/null)
|
|
44
|
+
STALE_PHASE=$(jq -r '.currentPhase // "?"' "$NEWEST" 2>/dev/null)
|
|
45
|
+
fi
|
|
46
|
+
fi
|
|
47
|
+
fi
|
|
48
|
+
|
|
49
|
+
REVIEW_DUE=0
|
|
50
|
+
OPEN_OBS=0
|
|
51
|
+
if [ -d "$PIPELINE_MEM/observations" ]; then
|
|
52
|
+
OPEN_OBS=$(grep -l '^status: open' "$PIPELINE_MEM/observations"/*.md 2>/dev/null | wc -l | tr -d ' ')
|
|
53
|
+
LAST_REVIEW_FILE="$PIPELINE_MEM/last-review-date.txt"
|
|
54
|
+
LAST=$(cat "$LAST_REVIEW_FILE" 2>/dev/null || echo "never")
|
|
55
|
+
if [ "${OPEN_OBS:-0}" -gt 0 ]; then
|
|
56
|
+
if [ "$LAST" = "never" ] || [ -z "$LAST" ]; then
|
|
57
|
+
REVIEW_DUE=1
|
|
58
|
+
else
|
|
59
|
+
# Date arithmetic without GNU date: compare epoch seconds, and treat an
|
|
60
|
+
# unparseable stamp as "due" rather than silently never reviewing again.
|
|
61
|
+
LAST_EPOCH=$(date -j -f "%Y-%m-%d" "$LAST" +%s 2>/dev/null \
|
|
62
|
+
|| date -d "$LAST" +%s 2>/dev/null || echo "")
|
|
63
|
+
if [ -z "$LAST_EPOCH" ]; then
|
|
64
|
+
REVIEW_DUE=1
|
|
65
|
+
else
|
|
66
|
+
AGE_DAYS=$(( ( $(date +%s) - LAST_EPOCH ) / 86400 ))
|
|
67
|
+
[ "$AGE_DAYS" -ge "$REVIEW_DAYS" ] && REVIEW_DUE=1
|
|
68
|
+
fi
|
|
69
|
+
fi
|
|
70
|
+
fi
|
|
71
|
+
fi
|
|
72
|
+
|
|
73
|
+
if [ "$JSON" -eq 1 ]; then
|
|
74
|
+
printf '{"staleTask":"%s","stalePhase":"%s","openObservations":%s,"reviewDue":%s}\n' \
|
|
75
|
+
"$STALE_TASK" "$STALE_PHASE" "${OPEN_OBS:-0}" "$REVIEW_DUE"
|
|
76
|
+
exit 0
|
|
77
|
+
fi
|
|
78
|
+
|
|
79
|
+
if [ -n "$STALE_TASK" ]; then
|
|
80
|
+
printf 'multi-agent: %s stopped at Phase %s - `/multi-agent:resume` to continue, `/multi-agent:kill` to drop it.\n' \
|
|
81
|
+
"$STALE_TASK" "$STALE_PHASE"
|
|
82
|
+
fi
|
|
83
|
+
if [ "$REVIEW_DUE" -eq 1 ]; then
|
|
84
|
+
printf 'multi-agent: %s open pipeline observation(s), last review %s+ days ago - `/multi-agent:refactor backlog` when convenient.\n' \
|
|
85
|
+
"$OPEN_OBS" "$REVIEW_DAYS"
|
|
86
|
+
fi
|
|
87
|
+
exit 0
|
|
@@ -0,0 +1,283 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* @file crush-json.mjs - summarise a JSON payload instead of truncating it.
|
|
4
|
+
*
|
|
5
|
+
* WHY THIS EXISTS
|
|
6
|
+
*
|
|
7
|
+
* `offload-ref.sh` writes a big payload to a file and leaves a stub in its
|
|
8
|
+
* place: a pointer plus the last N lines. For a log that is the right stub - the
|
|
9
|
+
* tail is where the failure is. For JSON it is close to worthless: the last
|
|
10
|
+
* twenty lines of a 200-element array are the final element and some closing
|
|
11
|
+
* brackets, which say nothing about the 199 above them. The reader gets a
|
|
12
|
+
* pointer and no shape, so either they open the whole file (paying for what the
|
|
13
|
+
* offload was meant to avoid) or they proceed uninformed.
|
|
14
|
+
*
|
|
15
|
+
* So: keep the shape and the outliers, drop the repetition.
|
|
16
|
+
*
|
|
17
|
+
* WHAT IS ALWAYS KEPT, whatever the budget
|
|
18
|
+
*
|
|
19
|
+
* - anything whose text carries error, exception, failed, failure, critical,
|
|
20
|
+
* fatal, denied, timeout - a summary that averages away the one failing
|
|
21
|
+
* element is worse than no summary
|
|
22
|
+
* - numeric outliers beyond two standard deviations, and length outliers
|
|
23
|
+
* - the first and last elements, which is where a reader looks for the shape
|
|
24
|
+
*
|
|
25
|
+
* WHAT IS NEVER TOUCHED
|
|
26
|
+
*
|
|
27
|
+
* - anything that is not JSON. Invalid input is returned byte-identical.
|
|
28
|
+
* - arrays under 5 elements, and payloads under ~200 tokens: there is nothing
|
|
29
|
+
* to win and a summary reads worse than the thing itself.
|
|
30
|
+
* - code. Detected by extension and by content shape, and passed through.
|
|
31
|
+
*
|
|
32
|
+
* FAIL-OPEN, ALWAYS
|
|
33
|
+
*
|
|
34
|
+
* Any error, any parse failure, and - critically - any result that is not
|
|
35
|
+
* smaller than the input returns the ORIGINAL unchanged. A compressor that can
|
|
36
|
+
* grow its input is a compressor that will, on the one payload nobody tested.
|
|
37
|
+
*
|
|
38
|
+
* Usage:
|
|
39
|
+
* ./crush-json.mjs <file> summary on stdout
|
|
40
|
+
* ./crush-json.mjs < payload.json same, from stdin
|
|
41
|
+
* ./crush-json.mjs <file> --stats report the before/after sizes only
|
|
42
|
+
*
|
|
43
|
+
* Exit codes are the contract, because a caller cannot tell a summary from a
|
|
44
|
+
* passthrough by looking at the bytes:
|
|
45
|
+
* 0 crushed - the output is a summary
|
|
46
|
+
* 3 passed through - the output IS the input, unchanged
|
|
47
|
+
* 1 unusable arguments
|
|
48
|
+
*
|
|
49
|
+
* offload-ref.sh used to infer this by comparing lengths, and command
|
|
50
|
+
* substitution strips trailing newlines - so every non-JSON payload looked
|
|
51
|
+
* "shorter" and therefore crushed, and the stub would have carried the entire
|
|
52
|
+
* log it was meant to replace.
|
|
53
|
+
*/
|
|
54
|
+
|
|
55
|
+
import { readFileSync } from "node:fs";
|
|
56
|
+
|
|
57
|
+
const argv = process.argv.slice(2);
|
|
58
|
+
const STATS = argv.includes("--stats");
|
|
59
|
+
const file = argv.find((a) => !a.startsWith("--"));
|
|
60
|
+
|
|
61
|
+
const MIN_ARRAY = 5;
|
|
62
|
+
const MIN_CHARS = 800; // ~200 tokens
|
|
63
|
+
const SAMPLE = 3;
|
|
64
|
+
const ALWAYS_KEEP =
|
|
65
|
+
/\b(error|errors|exception|failed|failure|critical|fatal|denied|timeout|refused)\b/i;
|
|
66
|
+
|
|
67
|
+
function read() {
|
|
68
|
+
if (file) {
|
|
69
|
+
try {
|
|
70
|
+
return readFileSync(file, "utf8");
|
|
71
|
+
} catch {
|
|
72
|
+
return null;
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
try {
|
|
76
|
+
return readFileSync(0, "utf8");
|
|
77
|
+
} catch {
|
|
78
|
+
return "";
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/** Code is passed through: it has no repeated structure to summarise. */
|
|
83
|
+
function looksLikeCode(text) {
|
|
84
|
+
if (file && /\.(swift|kt|java|js|mjs|ts|tsx|py|rb|go|rs|c|h|cpp|sh)$/i.test(file)) return true;
|
|
85
|
+
const codeish =
|
|
86
|
+
/^\s*(import |func |class |struct |def |function |const |let |var |package |#include)/m;
|
|
87
|
+
return codeish.test(text) && !/^\s*[[{]/.test(text.trim());
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
const isPrimitive = (v) => v === null || ["string", "number", "boolean"].includes(typeof v);
|
|
91
|
+
|
|
92
|
+
function mean(ns) {
|
|
93
|
+
return ns.reduce((a, b) => a + b, 0) / ns.length;
|
|
94
|
+
}
|
|
95
|
+
function stdev(ns) {
|
|
96
|
+
const m = mean(ns);
|
|
97
|
+
return Math.sqrt(mean(ns.map((n) => (n - m) ** 2)));
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/** Indices that must survive whatever else is dropped. */
|
|
101
|
+
function protectedIndices(arr) {
|
|
102
|
+
const keep = new Set([0, arr.length - 1]);
|
|
103
|
+
const texts = arr.map((v) => (typeof v === "string" ? v : (JSON.stringify(v) ?? "")));
|
|
104
|
+
texts.forEach((t, i) => {
|
|
105
|
+
if (ALWAYS_KEEP.test(t)) keep.add(i);
|
|
106
|
+
});
|
|
107
|
+
|
|
108
|
+
const nums = arr.filter((v) => typeof v === "number");
|
|
109
|
+
if (nums.length >= MIN_ARRAY) {
|
|
110
|
+
const m = mean(nums);
|
|
111
|
+
const sd = stdev(nums);
|
|
112
|
+
if (sd > 0)
|
|
113
|
+
arr.forEach((v, i) => {
|
|
114
|
+
if (typeof v === "number" && Math.abs(v - m) > 2 * sd) keep.add(i);
|
|
115
|
+
});
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
const lens = texts.map((t) => t.length);
|
|
119
|
+
if (lens.length >= MIN_ARRAY) {
|
|
120
|
+
const m = mean(lens);
|
|
121
|
+
const sd = stdev(lens);
|
|
122
|
+
if (sd > 0)
|
|
123
|
+
lens.forEach((l, i) => {
|
|
124
|
+
if (Math.abs(l - m) > 2 * sd) keep.add(i);
|
|
125
|
+
});
|
|
126
|
+
}
|
|
127
|
+
return keep;
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
function crushNumberArray(arr, keep) {
|
|
131
|
+
const sorted = arr.slice().sort((a, b) => a - b);
|
|
132
|
+
const mid = Math.floor(sorted.length / 2);
|
|
133
|
+
return {
|
|
134
|
+
_crushed: `${arr.length} numbers`,
|
|
135
|
+
min: sorted[0],
|
|
136
|
+
max: sorted[sorted.length - 1],
|
|
137
|
+
mean: Number(mean(arr).toFixed(3)),
|
|
138
|
+
median: sorted.length % 2 ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2,
|
|
139
|
+
kept: [...keep].sort((a, b) => a - b).map((i) => arr[i]),
|
|
140
|
+
};
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
function crushStringArray(arr, keep) {
|
|
144
|
+
const counts = new Map();
|
|
145
|
+
for (const s of arr) counts.set(s, (counts.get(s) || 0) + 1);
|
|
146
|
+
const distinct = [...counts.entries()].sort((a, b) => b[1] - a[1]);
|
|
147
|
+
return {
|
|
148
|
+
_crushed: `${arr.length} strings, ${counts.size} distinct`,
|
|
149
|
+
top: distinct.slice(0, SAMPLE).map(([v, n]) => (n > 1 ? `${v} (x${n})` : v)),
|
|
150
|
+
kept: [...keep].sort((a, b) => a - b).map((i) => arr[i]),
|
|
151
|
+
};
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/**
|
|
155
|
+
* Object arrays are the common shape (a gh api page, an accessibility tree, a
|
|
156
|
+
* findings list). Fields whose value is the same everywhere are stated once;
|
|
157
|
+
* fields that vary are sampled.
|
|
158
|
+
*/
|
|
159
|
+
function crushObjectArray(arr, keep) {
|
|
160
|
+
const fields = new Map();
|
|
161
|
+
for (const o of arr) {
|
|
162
|
+
for (const [k, v] of Object.entries(o || {})) {
|
|
163
|
+
if (!fields.has(k)) fields.set(k, { vals: new Set(), seen: 0 });
|
|
164
|
+
const f = fields.get(k);
|
|
165
|
+
f.seen += 1;
|
|
166
|
+
if (f.vals.size <= 4) f.vals.add(isPrimitive(v) ? String(v) : "«nested»");
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
const constant = {};
|
|
170
|
+
const varying = [];
|
|
171
|
+
const sparse = {};
|
|
172
|
+
for (const [k, { vals, seen }] of fields) {
|
|
173
|
+
// "Constant" has to mean constant ACROSS THE WHOLE ARRAY. One value seen
|
|
174
|
+
// once is not a property of 60 rows: a `message` field that existed only on
|
|
175
|
+
// the single failing element was being reported as though every row carried
|
|
176
|
+
// it, which is the exact way a summary misleads worse than a truncation.
|
|
177
|
+
if (vals.size === 1 && seen === arr.length) constant[k] = [...vals][0];
|
|
178
|
+
else if (seen < arr.length) sparse[k] = `${seen}/${arr.length}`;
|
|
179
|
+
else varying.push(k);
|
|
180
|
+
}
|
|
181
|
+
const out = {
|
|
182
|
+
_crushed: `${arr.length} objects`,
|
|
183
|
+
constantFields: constant,
|
|
184
|
+
varyingFields: varying,
|
|
185
|
+
kept: [...keep].sort((a, b) => a - b).map((i) => arr[i]),
|
|
186
|
+
};
|
|
187
|
+
// Fields only some elements carry, with how many carry them - the presence
|
|
188
|
+
// itself is often the signal (only failures have `message`).
|
|
189
|
+
if (Object.keys(sparse).length) out.partialFields = sparse;
|
|
190
|
+
return out;
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
function crushValue(v) {
|
|
194
|
+
if (Array.isArray(v)) {
|
|
195
|
+
if (v.length < MIN_ARRAY) return v.map(crushValue);
|
|
196
|
+
const keep = protectedIndices(v);
|
|
197
|
+
if (v.every((x) => typeof x === "number")) return crushNumberArray(v, keep);
|
|
198
|
+
if (v.every((x) => typeof x === "string")) return crushStringArray(v, keep);
|
|
199
|
+
if (v.every((x) => typeof x === "boolean")) return v; // nothing to win
|
|
200
|
+
if (v.every((x) => x && typeof x === "object" && !Array.isArray(x))) {
|
|
201
|
+
return crushObjectArray(v, keep);
|
|
202
|
+
}
|
|
203
|
+
return {
|
|
204
|
+
_crushed: `${v.length} mixed items`,
|
|
205
|
+
kept: [...keep].sort((a, b) => a - b).map((i) => crushValue(v[i])),
|
|
206
|
+
};
|
|
207
|
+
}
|
|
208
|
+
if (v && typeof v === "object") {
|
|
209
|
+
const out = {};
|
|
210
|
+
for (const [k, val] of Object.entries(v)) out[k] = crushValue(val);
|
|
211
|
+
return out;
|
|
212
|
+
}
|
|
213
|
+
return v;
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
const original = read();
|
|
217
|
+
if (original === null) {
|
|
218
|
+
process.stderr.write(`ERR: cannot read ${file}\n`);
|
|
219
|
+
process.exit(1);
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
function emit(text, note) {
|
|
223
|
+
const crushed = note === "crushed";
|
|
224
|
+
if (STATS) {
|
|
225
|
+
process.stdout.write(
|
|
226
|
+
JSON.stringify({
|
|
227
|
+
before: original.length,
|
|
228
|
+
after: text.length,
|
|
229
|
+
crushed,
|
|
230
|
+
reason: note || null,
|
|
231
|
+
}) + "\n",
|
|
232
|
+
);
|
|
233
|
+
} else if (crushed) {
|
|
234
|
+
process.stdout.write(text.endsWith("\n") ? text : text + "\n");
|
|
235
|
+
} else {
|
|
236
|
+
// Passthrough means byte-identical. Adding a trailing newline the input did
|
|
237
|
+
// not have is a small lie, and it is the kind that makes a caller's diff
|
|
238
|
+
// disagree with its own contract.
|
|
239
|
+
process.stdout.write(text);
|
|
240
|
+
}
|
|
241
|
+
// `process.exitCode`, never `process.exit()`. stdout to a PIPE is async, and
|
|
242
|
+
// exiting immediately after a large write truncates it at the pipe buffer -
|
|
243
|
+
// a 271 KB summary arrived as exactly 65536 bytes, silently, and only when
|
|
244
|
+
// the caller piped rather than redirected. Setting the code lets the write
|
|
245
|
+
// drain and the process end on its own.
|
|
246
|
+
process.exitCode = crushed ? 0 : 3;
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
// The decision chain lives in a function so each branch can RETURN. emit() no
|
|
250
|
+
// longer exits - it cannot, without truncating a large write to a pipe - so a
|
|
251
|
+
// straight-line version would fall through every branch and crush a payload it
|
|
252
|
+
// had already decided to pass through.
|
|
253
|
+
function main() {
|
|
254
|
+
if (!original.trim()) return emit(original, "empty");
|
|
255
|
+
if (original.length < MIN_CHARS) return emit(original, "below-threshold");
|
|
256
|
+
if (looksLikeCode(original)) return emit(original, "code-passthrough");
|
|
257
|
+
|
|
258
|
+
let parsed;
|
|
259
|
+
try {
|
|
260
|
+
parsed = JSON.parse(original);
|
|
261
|
+
} catch {
|
|
262
|
+
return emit(original, "not-json");
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
let out;
|
|
266
|
+
try {
|
|
267
|
+
out = JSON.stringify(crushValue(parsed), null, 2);
|
|
268
|
+
} catch {
|
|
269
|
+
return emit(original, "crush-threw");
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
// Never hand back something bigger than what came in - and never claim a
|
|
273
|
+
// crush for a saving too small to matter. A deeply nested payload with few
|
|
274
|
+
// arrays came back one byte shorter, which is not a summary, it is a rewrite
|
|
275
|
+
// of a readable thing for nothing.
|
|
276
|
+
const MIN_GAIN = 0.15;
|
|
277
|
+
if (!out || out.length >= original.length * (1 - MIN_GAIN)) {
|
|
278
|
+
return emit(original, "no-gain");
|
|
279
|
+
}
|
|
280
|
+
return emit(out, "crushed");
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
main();
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
#
|
|
3
|
+
# firebase-app-discovery.sh - read the Firebase app ids a repo already carries.
|
|
4
|
+
#
|
|
5
|
+
# Every Crashlytics call needs the opaque appId (1:<number>:<platform>:<hex>), and
|
|
6
|
+
# a console URL only ever carries the bundle id. fetch-crashlytics.sh resolves the
|
|
7
|
+
# pair through the Firebase Management API on every run, which works and costs a
|
|
8
|
+
# round trip plus a resolved project. The repo has already been told the answer:
|
|
9
|
+
# GoogleService-Info*.plist (iOS) and google-services.json (Android) are generated
|
|
10
|
+
# by the console and carry both ids.
|
|
11
|
+
#
|
|
12
|
+
# Usage:
|
|
13
|
+
# ./firebase-app-discovery.sh [repo-dir] # table
|
|
14
|
+
# ./firebase-app-discovery.sh [repo-dir] --json # {"accounts":[...]}
|
|
15
|
+
#
|
|
16
|
+
# --json prints entries shaped for prefs.global.firebase.accounts[]: one per
|
|
17
|
+
# projectId, each with an apps[] of {bundleId, appId, platform}. keychainKey is
|
|
18
|
+
# left out on purpose - which credential covers a project is the user's mapping to
|
|
19
|
+
# make, not this script's to guess.
|
|
20
|
+
#
|
|
21
|
+
# A repo with several targets has several plists and they do not all point at one
|
|
22
|
+
# Firebase project: every match is read, never just the first. Build outputs are
|
|
23
|
+
# skipped, because a copied plist under DerivedData or build/ is the same app
|
|
24
|
+
# counted twice.
|
|
25
|
+
#
|
|
26
|
+
# Exit 0 always: finding nothing is an answer, not a failure.
|
|
27
|
+
|
|
28
|
+
set -uo pipefail
|
|
29
|
+
|
|
30
|
+
DIR="."
|
|
31
|
+
MODE="table"
|
|
32
|
+
for a in "$@"; do
|
|
33
|
+
case "$a" in
|
|
34
|
+
--json) MODE="json" ;;
|
|
35
|
+
-h|--help) echo "usage: $0 [repo-dir] [--json]" >&2; exit 0 ;;
|
|
36
|
+
*) DIR="$a" ;;
|
|
37
|
+
esac
|
|
38
|
+
done
|
|
39
|
+
|
|
40
|
+
if [ ! -d "$DIR" ]; then
|
|
41
|
+
echo "ERR: not a directory: $DIR" >&2
|
|
42
|
+
exit 0
|
|
43
|
+
fi
|
|
44
|
+
|
|
45
|
+
FILES=$(find "$DIR" \
|
|
46
|
+
\( -name node_modules -o -name Pods -o -name .build -o -name DerivedData \
|
|
47
|
+
-o -name build -o -name .git -o -name .next \) -prune -o \
|
|
48
|
+
\( -name 'GoogleService-Info*.plist' -o -name 'google-services.json' \) -print 2>/dev/null)
|
|
49
|
+
|
|
50
|
+
# The file list travels as an env var, not on stdin: `python3 - <<SCRIPT` already
|
|
51
|
+
# takes its program from stdin, and a second redirection silently wins, feeding
|
|
52
|
+
# the interpreter the paths as if they were source.
|
|
53
|
+
MODE="$MODE" FB_FILES="$FILES" python3 - "$DIR" <<'PY'
|
|
54
|
+
import json, os, plistlib, sys
|
|
55
|
+
|
|
56
|
+
repo = sys.argv[1]
|
|
57
|
+
paths = [p for p in os.environ.get("FB_FILES", "").split("\n") if p.strip()]
|
|
58
|
+
|
|
59
|
+
# projectId -> {bundleId: (appId, platform)}; a dict per project because the same
|
|
60
|
+
# target can appear twice (a Debug and a Release plist naming one app), and the
|
|
61
|
+
# second read must not double the row.
|
|
62
|
+
projects = {}
|
|
63
|
+
|
|
64
|
+
def record(project, bundle, app_id, platform):
|
|
65
|
+
if not (project and bundle and app_id):
|
|
66
|
+
return
|
|
67
|
+
projects.setdefault(project, {})[bundle] = (app_id, platform)
|
|
68
|
+
|
|
69
|
+
for path in paths:
|
|
70
|
+
try:
|
|
71
|
+
if path.endswith(".plist"):
|
|
72
|
+
with open(path, "rb") as fh:
|
|
73
|
+
d = plistlib.load(fh)
|
|
74
|
+
app_id = d.get("GOOGLE_APP_ID") or ""
|
|
75
|
+
platform = "android" if ":android:" in app_id else "ios"
|
|
76
|
+
record(d.get("PROJECT_ID"), d.get("BUNDLE_ID"), app_id, platform)
|
|
77
|
+
else:
|
|
78
|
+
with open(path, encoding="utf-8") as fh:
|
|
79
|
+
d = json.load(fh)
|
|
80
|
+
project = ((d.get("project_info") or {}).get("project_id")) or ""
|
|
81
|
+
for client in d.get("client") or []:
|
|
82
|
+
info = client.get("client_info") or {}
|
|
83
|
+
app_id = info.get("mobilesdk_app_id") or ""
|
|
84
|
+
bundle = ((info.get("android_client_info") or {}).get("package_name")) or ""
|
|
85
|
+
record(project, bundle, app_id, "android")
|
|
86
|
+
except Exception:
|
|
87
|
+
# A malformed or unreadable file is one app not discovered, never a reason
|
|
88
|
+
# to abandon the ones that parsed.
|
|
89
|
+
continue
|
|
90
|
+
|
|
91
|
+
accounts = [
|
|
92
|
+
{
|
|
93
|
+
"projectId": pid,
|
|
94
|
+
"apps": [
|
|
95
|
+
{"bundleId": b, "appId": a, "platform": pf}
|
|
96
|
+
for b, (a, pf) in sorted(apps.items())
|
|
97
|
+
],
|
|
98
|
+
}
|
|
99
|
+
for pid, apps in sorted(projects.items())
|
|
100
|
+
]
|
|
101
|
+
|
|
102
|
+
if os.environ.get("MODE") == "json":
|
|
103
|
+
print(json.dumps({"accounts": accounts}, indent=2))
|
|
104
|
+
sys.exit(0)
|
|
105
|
+
|
|
106
|
+
if not accounts:
|
|
107
|
+
print("no Firebase config found under %s" % repo)
|
|
108
|
+
sys.exit(0)
|
|
109
|
+
|
|
110
|
+
for acc in accounts:
|
|
111
|
+
print(acc["projectId"])
|
|
112
|
+
for app in acc["apps"]:
|
|
113
|
+
print(" %-8s %-45s %s" % (app["platform"], app["bundleId"], app["appId"]))
|
|
114
|
+
PY
|
|
@@ -39,13 +39,15 @@ case "$CHOICE" in
|
|
|
39
39
|
echo "Hata: Dosya bulunamadı: $JSON_PATH"
|
|
40
40
|
exit 1
|
|
41
41
|
fi
|
|
42
|
-
# JSON
|
|
43
|
-
|
|
42
|
+
# JSON olduğu gibi saklanır. Kimlik deposu çok satırlı değeri
|
|
43
|
+
# bayt bayt geri verir; base64 sarmalı okuma tarafında çözülmeyen
|
|
44
|
+
# bir katman ekliyordu.
|
|
45
|
+
SECRET=$(cat "$JSON_PATH")
|
|
44
46
|
if [ -z "$SECRET" ]; then
|
|
45
47
|
echo "Hata: Dosya okunamadı."
|
|
46
48
|
exit 1
|
|
47
49
|
fi
|
|
48
|
-
echo "JSON
|
|
50
|
+
echo "JSON olduğu gibi saklanacak."
|
|
49
51
|
;;
|
|
50
52
|
*)
|
|
51
53
|
echo "Geçersiz seçim."
|
|
@@ -89,11 +91,6 @@ if printf '%s' "$SECRET" | "$CRED" set "$SERVICE_NAME" -; then
|
|
|
89
91
|
echo ""
|
|
90
92
|
echo "Okumak icin:"
|
|
91
93
|
echo " $CRED get \"$SERVICE_NAME\""
|
|
92
|
-
if [ "$CHOICE" = "2" ]; then
|
|
93
|
-
echo ""
|
|
94
|
-
echo "JSON'a geri cevirmek icin:"
|
|
95
|
-
echo " $CRED get \"$SERVICE_NAME\" | base64 -d"
|
|
96
|
-
fi
|
|
97
94
|
else
|
|
98
95
|
echo "Hata: Kaydedilemedi."
|
|
99
96
|
exit 1
|