crewly 1.20.447 → 1.20.449
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/config/skills/agent/checkup-analyse/SKILL.md +99 -0
- package/config/skills/agent/checkup-analyse/analyse.mjs +603 -0
- package/config/skills/agent/checkup-analyse/execute.sh +34 -0
- package/config/skills/agent/checkup-analyse/execute.test.sh +126 -0
- package/config/skills/agent/ingest-exports/SKILL.md +97 -0
- package/config/skills/agent/ingest-exports/execute.sh +35 -0
- package/config/skills/agent/ingest-exports/execute.test.sh +82 -0
- package/config/skills/agent/ingest-exports/fixtures/jackrabbit-enrollments.csv +4 -0
- package/config/skills/agent/ingest-exports/fixtures/jotform-submissions.csv +4 -0
- package/config/skills/agent/ingest-exports/fixtures/mailchimp-audience.csv +3 -0
- package/config/skills/agent/ingest-exports/fixtures/quickbooks-invoices.csv +4 -0
- package/config/skills/agent/ingest-exports/fixtures/square-customers.csv +4 -0
- package/config/skills/agent/ingest-exports/fixtures/square-transactions.csv +5 -0
- package/config/skills/agent/ingest-exports/fixtures/unknown-report.csv +4 -0
- package/config/skills/agent/ingest-exports/ingest.mjs +368 -0
- package/dist/backend/build-info.json +2 -2
- package/package.json +1 -1
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
#!/bin/bash
|
|
2
|
+
# Tests for checkup-analyse — run with: bash execute.test.sh
|
|
3
|
+
# Offline: runs ingest-exports on its fixtures first, then analyses the result. Needs node and jq.
|
|
4
|
+
set -u
|
|
5
|
+
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
6
|
+
EXEC="$HERE/execute.sh"
|
|
7
|
+
INGEST="$HERE/../ingest-exports/execute.sh"
|
|
8
|
+
FX="$HERE/../ingest-exports/fixtures"
|
|
9
|
+
PASS=0; FAIL=0
|
|
10
|
+
check() {
|
|
11
|
+
local name="$1"; local got="$2"; local want="$3"
|
|
12
|
+
if [ "$got" = "$want" ]; then PASS=$((PASS+1)); else FAIL=$((FAIL+1)); echo "FAIL: $name"; echo " want: $want"; echo " got: $got"; fi
|
|
13
|
+
}
|
|
14
|
+
T="$(mktemp -d)"; trap 'rm -rf "$T"' EXIT
|
|
15
|
+
bash "$INGEST" --in "$FX" --out "$T/ingest" >/dev/null || { echo "ingest-exports failed; cannot test"; exit 1; }
|
|
16
|
+
bash "$INGEST" --in "$FX" --out "$T/ingest-hashed" --hash-names >/dev/null
|
|
17
|
+
|
|
18
|
+
# 1. Default as-of = last activity; period = activity span, not contact creation dates
|
|
19
|
+
OUT="$(bash "$EXEC" --in "$T/ingest" --out "$T/an")"; RC=$?
|
|
20
|
+
check "exit 0" "$RC" "0"
|
|
21
|
+
check "success" "$(printf '%s' "$OUT" | jq -r '.success')" "true"
|
|
22
|
+
check "as-of = last activity" "$(printf '%s' "$OUT" | jq -r '.asOf')" "2026-09-20"
|
|
23
|
+
check "period from first sign-up, not mailing-list opt-in" "$(printf '%s' "$OUT" | jq -r '.period.from')" "2026-08-28"
|
|
24
|
+
check "period days" "$(printf '%s' "$OUT" | jq -r '.period.days')" "23"
|
|
25
|
+
check "people linked across files" "$(printf '%s' "$OUT" | jq -r '.people')" "8"
|
|
26
|
+
check "systems exclude the unmapped file" "$(printf '%s' "$OUT" | jq -c '.systems')" '["jackrabbit","jotform","mailchimp","quickbooks","square"]'
|
|
27
|
+
check "breaks.json written" "$([ -f "$T/an/breaks.json" ] && echo yes || echo no)" "yes"
|
|
28
|
+
check "report.md written" "$([ -f "$T/an/report.md" ] && echo yes || echo no)" "yes"
|
|
29
|
+
cnt() { jq -r --arg t "$1" '.breaks[] | select(.type==$t) | .count' "$T/an/breaks.json"; }
|
|
30
|
+
item() { jq -r --arg t "$1" ".breaks[] | select(.type==\$t) | $2" "$T/an/breaks.json"; }
|
|
31
|
+
|
|
32
|
+
# 2. Each break finds exactly the planted case
|
|
33
|
+
check "inquiry with no follow-through = Tom only (Alex and Priya signed up)" "$(cnt inquiry_no_follow_through)" "1"
|
|
34
|
+
check " who" "$(item inquiry_no_follow_through '.items[0].person')" "Tom Baker"
|
|
35
|
+
check "signed up unpaid = Sam Rivera (Sofia's family paid on 09-02)" "$(cnt signed_up_unpaid)" "1"
|
|
36
|
+
check " who" "$(item signed_up_unpaid '.items[0].person')" "Sam Rivera"
|
|
37
|
+
check " fee" "$(item signed_up_unpaid '.items[0].fee')" "180"
|
|
38
|
+
check " outstanding" "$(item signed_up_unpaid '.estimate.outstanding')" "180"
|
|
39
|
+
check "paid but on no roster = Daniel (walk-in with no name is skipped)" "$(cnt paid_not_listed)" "1"
|
|
40
|
+
check " who" "$(item paid_not_listed '.items[0].person')" "Daniel Chen"
|
|
41
|
+
check "dropped = Anika" "$(cnt dropped)" "1"
|
|
42
|
+
check " dropped date from end date" "$(item dropped '.items[0].droppedAt')" "2026-09-22"
|
|
43
|
+
check "lapsed = none at default as-of" "$(cnt lapsed)" "0"
|
|
44
|
+
check "overdue invoices = none at default as-of (due 09-30 and 10-15)" "$(cnt overdue_invoices)" "0"
|
|
45
|
+
check " open not yet due" "$(item overdue_invoices '.estimate.openNotYetDue')" "2"
|
|
46
|
+
check "double entry = 4 people in 2+ systems" "$(cnt double_entry)" "4"
|
|
47
|
+
check " adult label, not the child's name" "$(item double_entry '.items[0].person')" "Maria Lopez"
|
|
48
|
+
check " family in three systems" "$(item double_entry '.items[0].systems | join(",")')" "jackrabbit,mailchimp,square"
|
|
49
|
+
check " child vs parent name is not a conflict" "$(item double_entry '.items[0].conflicts | length')" "0"
|
|
50
|
+
check " phone missing in mailchimp noted" "$(item double_entry '.items[3].conflicts[0]')" "phone missing in mailchimp"
|
|
51
|
+
check " extra entries (2+1+2+1)" "$(item double_entry '.estimate.extraEntries')" "6"
|
|
52
|
+
check "not on mailing list = 3 customers" "$(cnt not_on_mailing_list)" "3"
|
|
53
|
+
check " Maria is on it" "$(item not_on_mailing_list '[.items[].person] | index("Maria Lopez")')" "null"
|
|
54
|
+
check "anonymous walk-in counted" "$(jq -r '.stats.anonymousRows.payment' "$T/an/breaks.json")" "1"
|
|
55
|
+
check "unmapped rows ignored, not dropped silently" "$(jq -r '.stats.ignoredOtherRows' "$T/an/breaks.json")" "3"
|
|
56
|
+
check "refunds counted" "$(jq -c '.stats.refunds' "$T/an/breaks.json")" '{"count":1,"amount":49}'
|
|
57
|
+
|
|
58
|
+
# 3. Estimates carry a formula and are marked estimates; avg customer value comes from the data
|
|
59
|
+
check "avg customer value from data" "$(jq -r '.assumptions.avgCustomerValue' "$T/an/breaks.json")" "113"
|
|
60
|
+
check "avg source named" "$(jq -r '.stats.avgCustomerValueFrom' "$T/an/breaks.json" | cut -c1-7)" "mean of"
|
|
61
|
+
check "every estimate marked" "$(jq -r '[.breaks[].estimate.isEstimate] | all' "$T/an/breaks.json")" "true"
|
|
62
|
+
check "every estimate has a formula" "$(jq -r '[.breaks[].estimate.formula | length > 20] | all' "$T/an/breaks.json")" "true"
|
|
63
|
+
check "unpaid $/month = outstanding over 1 month" "$(item signed_up_unpaid '.estimate.dollarsPerMonth')" "180"
|
|
64
|
+
check "top 3 ranked by $/month" "$(jq -c '.top3' "$T/an/breaks.json")" '["signed_up_unpaid","dropped","inquiry_no_follow_through"]'
|
|
65
|
+
check "totals add up" "$(jq -r '.totals.dollarsPerMonth' "$T/an/breaks.json")" "$(jq -r '[.breaks[].estimate.dollarsPerMonth] | add' "$T/an/breaks.json")"
|
|
66
|
+
check "every break says what an agent could do" "$(jq -r '[.breaks[].whatAnAgentCouldDo | length > 20] | all' "$T/an/breaks.json")" "true"
|
|
67
|
+
|
|
68
|
+
# 4. report.md is the owner-facing text
|
|
69
|
+
check "report title" "$(head -1 "$T/an/report.md")" "# Check-up: where the apps disagree"
|
|
70
|
+
check "report has top 3" "$(grep -c '^### [123]\. ' "$T/an/report.md")" "3"
|
|
71
|
+
check "report says estimate" "$(grep -c -i 'estimate' "$T/an/report.md" | awk '{print ($1>0)?"yes":"no"}')" "yes"
|
|
72
|
+
check "report lists assumptions" "$(grep -c '^- hourlyValue: 30' "$T/an/report.md")" "1"
|
|
73
|
+
check "report lists files" "$(grep -c '^- square-transactions.csv: square payment, 4 rows' "$T/an/report.md")" "1"
|
|
74
|
+
|
|
75
|
+
# 5. --as-of moves overdue and lapsed; --assume changes the inputs
|
|
76
|
+
LATE="$(bash "$EXEC" --in "$T/ingest" --as-of 2026-10-26 --out "$T/late")"
|
|
77
|
+
check "as-of honoured" "$(printf '%s' "$LATE" | jq -r '.asOf')" "2026-10-26"
|
|
78
|
+
check "two invoices overdue on 10-26" "$(printf '%s' "$LATE" | jq -r '.breaks[] | select(.type=="overdue_invoices") | .count')" "2"
|
|
79
|
+
check " outstanding" "$(jq -r '.breaks[] | select(.type=="overdue_invoices") | .estimate.outstanding' "$T/late/breaks.json")" "3391"
|
|
80
|
+
check " days overdue" "$(jq -r '.breaks[] | select(.type=="overdue_invoices") | .items[0].daysOverdue' "$T/late/breaks.json")" "26"
|
|
81
|
+
check " overdue is now top" "$(printf '%s' "$LATE" | jq -r '.top3[0]')" "overdue_invoices"
|
|
82
|
+
check "still nobody lapsed at 90 days" "$(printf '%s' "$LATE" | jq -r '.breaks[] | select(.type=="lapsed") | .count')" "0"
|
|
83
|
+
LAPSED="$(bash "$EXEC" --in "$T/ingest" --as-of 2026-10-26 --assume lapsedDays=30 --assume hourlyValue=60)"
|
|
84
|
+
check "lapsedDays=30 → 4 customers lapsed" "$(printf '%s' "$LAPSED" | jq -r '.breaks[] | select(.type=="lapsed") | .count')" "4"
|
|
85
|
+
FAR="$(bash "$EXEC" --in "$T/ingest" --as-of 2027-01-15 --out "$T/far")"
|
|
86
|
+
check "a year on, every customer is lapsed" "$(printf '%s' "$FAR" | jq -r '.breaks[] | select(.type=="lapsed") | .count')" "4"
|
|
87
|
+
check "lapsed item has days since" "$(jq -r '.breaks[] | select(.type=="lapsed") | .items[0].daysSince > 100' "$T/far/breaks.json")" "true"
|
|
88
|
+
check "hourlyValue assumption applied" "$(bash "$EXEC" --in "$T/ingest" --assume hourlyValue=60 --out "$T/hv" >/dev/null; jq -r '.assumptions.hourlyValue' "$T/hv/breaks.json")" "60"
|
|
89
|
+
|
|
90
|
+
# 6. Hashed input: same counts, no names anywhere
|
|
91
|
+
HOUT="$(bash "$EXEC" --in "$T/ingest-hashed" --out "$T/anh")"
|
|
92
|
+
check "hashed: same break counts" "$(printf '%s' "$HOUT" | jq -c '[.breaks[].count]')" "$(printf '%s' "$OUT" | jq -c '[.breaks[].count]')"
|
|
93
|
+
check "hashed: no plain names in breaks.json" "$(grep -c -E "Lopez|Rivera|Baker|Chen|Natarajan|example\.test|555 010" "$T/anh/breaks.json")" "0"
|
|
94
|
+
check "hashed: no plain names in report.md" "$(grep -c -E "Lopez|Rivera|Baker|Chen|Natarajan" "$T/anh/report.md")" "0"
|
|
95
|
+
check "hashed: person label is a hash" "$(jq -r '.breaks[] | select(.type=="signed_up_unpaid") | .items[0].person' "$T/anh/breaks.json" | cut -c1-2)" "h:"
|
|
96
|
+
|
|
97
|
+
# 7. Partial data: notes say which checks could not run
|
|
98
|
+
mkdir -p "$T/partial"; cp "$T/ingest/square-transactions.json" "$T/ingest/square-customers.json" "$T/partial/"
|
|
99
|
+
POUT="$(bash "$EXEC" --in "$T/partial")"
|
|
100
|
+
check "partial: exits 0" "$?" "0"
|
|
101
|
+
check "partial: roster check skipped with a note" "$(printf '%s' "$POUT" | jq -r '.notes | map(select(test("paid but not listed"))) | length')" "1"
|
|
102
|
+
check "partial: mailing-list check skipped with a note" "$(printf '%s' "$POUT" | jq -r '.notes | map(select(test("mailing list"))) | length')" "1"
|
|
103
|
+
check "partial: paid_not_listed stays 0" "$(printf '%s' "$POUT" | jq -r '.breaks[] | select(.type=="paid_not_listed") | .count')" "0"
|
|
104
|
+
check "single file input works" "$(bash "$EXEC" --in "$T/ingest/quickbooks-invoices.json" --as-of 2026-10-26 | jq -r '.breaks[] | select(.type=="overdue_invoices") | .count')" "2"
|
|
105
|
+
|
|
106
|
+
# 8. JSON input
|
|
107
|
+
JOUT="$(bash "$EXEC" "{\"in\":\"$T/ingest\",\"out\":\"$T/json\",\"asOf\":\"2026-10-26\",\"assume\":{\"lapsedDays\":30}}")"
|
|
108
|
+
check "json input: as-of" "$(printf '%s' "$JOUT" | jq -r '.asOf')" "2026-10-26"
|
|
109
|
+
check "json input: assumption" "$(jq -r '.assumptions.lapsedDays' "$T/json/breaks.json")" "30"
|
|
110
|
+
check "json input: out written" "$([ -f "$T/json/report.md" ] && echo yes || echo no)" "yes"
|
|
111
|
+
|
|
112
|
+
# 9. Errors
|
|
113
|
+
bash "$EXEC" --in "$T/nope" >/dev/null 2>&1; check "missing input exits 2" "$?" "2"
|
|
114
|
+
mkdir -p "$T/empty"; bash "$EXEC" --in "$T/empty" >/dev/null 2>&1; check "empty folder exits 2" "$?" "2"
|
|
115
|
+
bash "$EXEC" >/dev/null 2>&1; check "no args exits 1" "$?" "1"
|
|
116
|
+
bash "$EXEC" --in "$T/ingest" --as-of 10/26/2026 >/dev/null 2>&1; check "bad as-of exits 1" "$?" "1"
|
|
117
|
+
bash "$EXEC" --in "$T/ingest" --assume nope=3 >/dev/null 2>&1; check "unknown assumption exits 1" "$?" "1"
|
|
118
|
+
bash "$EXEC" --in "$T/ingest" --assume hourlyValue=lots >/dev/null 2>&1; check "non-number assumption exits 1" "$?" "1"
|
|
119
|
+
printf 'not json' > "$T/ingest/broken.json"
|
|
120
|
+
BOUT="$(bash "$EXEC" --in "$T/ingest")"; RC=$?
|
|
121
|
+
check "unreadable file → exit 2 but the rest is analysed" "$RC" "2"
|
|
122
|
+
check " problem named" "$(printf '%s' "$BOUT" | jq -r '.problems[0]' | cut -c1-11)" "broken.json"
|
|
123
|
+
check " breaks still computed" "$(printf '%s' "$BOUT" | jq -r '.people')" "8"
|
|
124
|
+
|
|
125
|
+
echo "checkup-analyse: $PASS passed, $FAIL failed"
|
|
126
|
+
[ "$FAIL" -eq 0 ]
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: Ingest Exports
|
|
3
|
+
description: Load CSV exports from the tools a small business already uses (Square, QuickBooks, Mailchimp, Jotform, Jackrabbit, Mindbody, Calendly, Stripe) into one common schema (contact / booking / payment / invoice) with the original columns kept, and report rows and date range per file. Unknown CSVs still load with a "could not map" note. --hash-names keeps people's names, emails and phones out of what you store.
|
|
4
|
+
version: 1.0.0
|
|
5
|
+
category: analysis
|
|
6
|
+
skillType: claude-skill
|
|
7
|
+
assignableRoles:
|
|
8
|
+
- "*"
|
|
9
|
+
triggers:
|
|
10
|
+
- ingest exports
|
|
11
|
+
- load csv export
|
|
12
|
+
- import csv
|
|
13
|
+
- square export
|
|
14
|
+
- quickbooks export
|
|
15
|
+
- mailchimp export
|
|
16
|
+
- jotform export
|
|
17
|
+
- check-up data
|
|
18
|
+
tags:
|
|
19
|
+
- csv
|
|
20
|
+
- export
|
|
21
|
+
- check-up
|
|
22
|
+
- small-business
|
|
23
|
+
- square
|
|
24
|
+
- quickbooks
|
|
25
|
+
- mailchimp
|
|
26
|
+
- jotform
|
|
27
|
+
execution:
|
|
28
|
+
type: script
|
|
29
|
+
script:
|
|
30
|
+
file: execute.sh
|
|
31
|
+
interpreter: bash
|
|
32
|
+
timeoutMs: 120000
|
|
33
|
+
---
|
|
34
|
+
|
|
35
|
+
# Ingest Exports
|
|
36
|
+
|
|
37
|
+
Most tools a small business uses have an **Export to CSV** button even when
|
|
38
|
+
they have no API you can get a key for. This skill is the universal fallback
|
|
39
|
+
of the small-business check-up (crewly#1055): the owner exports, you ingest,
|
|
40
|
+
and every file ends up in the same shape so the analysis does not care where
|
|
41
|
+
it came from.
|
|
42
|
+
|
|
43
|
+
```bash
|
|
44
|
+
bash execute.sh --in ~/Downloads/exports --out "$CREWLY_HOME/checkup/<customer>" --hash-names
|
|
45
|
+
bash execute.sh --in square-transactions.csv --in invoices.csv --out /tmp/checkup
|
|
46
|
+
bash execute.sh --in exports/ --detect-only # what is each file? writes nothing
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
## Getting the files
|
|
50
|
+
|
|
51
|
+
- **Google Drive** (the usual drop point): `drive-search` for the folder the
|
|
52
|
+
owner shared, then `drive-read --id <id> --out <path>` per file. Save them
|
|
53
|
+
into one folder and pass that folder as `--in`.
|
|
54
|
+
- **App form / attachment**: files land under `$CREWLY_HOME/received/`;
|
|
55
|
+
pass the file paths.
|
|
56
|
+
|
|
57
|
+
Never ask the owner to paste exported rows into chat.
|
|
58
|
+
|
|
59
|
+
## What comes out
|
|
60
|
+
|
|
61
|
+
One JSON line, and under `--out` one `<name>.json` per file plus
|
|
62
|
+
`summary.json`:
|
|
63
|
+
|
|
64
|
+
```json
|
|
65
|
+
{"success":true,"files":[{"file":"square-transactions.csv","source":"square","kind":"payment","rows":4,
|
|
66
|
+
"dateRange":{"from":"2026-09-02","to":"2026-09-20"},"unmappedColumns":["Tax","Tip","Card Brand"],"notes":[]}],
|
|
67
|
+
"totals":{"files":1,"rows":4,"byKind":{"payment":4}}}
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
Every record has `kind` (`contact`, `booking`, `payment`, `invoice`, or
|
|
71
|
+
`other`), `source`, `file`, `row`, the mapped fields it could fill (`id`,
|
|
72
|
+
`name`, `email`, `phone`, `at`, `dueAt`, `endAt`, `amount`, `balance`,
|
|
73
|
+
`currency`, `status`, `item`, `tags`) and `original` — every exported column
|
|
74
|
+
as it was. Dates are ISO (`2026-09-02` or `2026-09-02T10:14:03`); amounts are
|
|
75
|
+
numbers (`"$1,250.00"` → `1250`).
|
|
76
|
+
|
|
77
|
+
Recognised exports: Square transactions and customers, QuickBooks Online
|
|
78
|
+
invoice and customer lists, Mailchimp audience, Jotform submissions,
|
|
79
|
+
Jackrabbit enrollments, Mindbody clients, Calendly scheduled events, Stripe
|
|
80
|
+
payments. A file that matches none still loads: `source: "unknown"`,
|
|
81
|
+
`kind: "other"`, records carry `original` only, and `notes` says
|
|
82
|
+
"could not map". Tell the owner which file that was and what tool it came
|
|
83
|
+
from; `--source <name>` forces a known mapping when the headers are close.
|
|
84
|
+
|
|
85
|
+
## People's data
|
|
86
|
+
|
|
87
|
+
Children's names, parents' emails and phone numbers are in most of these
|
|
88
|
+
exports. Run with `--hash-names` unless the analysis truly needs the names:
|
|
89
|
+
`name`, `email`, `phone` and every original column that looks personal become
|
|
90
|
+
a short stable hash (`h:3f1a…`), so the same person still counts once across
|
|
91
|
+
files but nobody's name is stored. Report counts, not names, and delete the
|
|
92
|
+
`--out` folder when the check-up ends.
|
|
93
|
+
|
|
94
|
+
## Report back
|
|
95
|
+
|
|
96
|
+
Per file, in one line each: tool, kind, rows, date range, and anything in
|
|
97
|
+
`notes`. Then the totals. Say plainly which files could not be mapped.
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
#!/bin/bash
|
|
2
|
+
# =============================================================================
|
|
3
|
+
# ingest-exports — load CSV exports from small-business tools (Square,
|
|
4
|
+
# QuickBooks, Mailchimp, Jotform, Jackrabbit, Mindbody, Calendly, Stripe)
|
|
5
|
+
# into one common schema for the small-business check-up (crewly#1061).
|
|
6
|
+
#
|
|
7
|
+
# Usage:
|
|
8
|
+
# bash execute.sh --in ~/Downloads/exports --out /tmp/checkup # every .csv in the folder
|
|
9
|
+
# bash execute.sh --in square.csv --in invoices.csv --out /tmp/checkup --hash-names
|
|
10
|
+
# bash execute.sh --in exports/ --detect-only # say what each file is, write nothing
|
|
11
|
+
# bash execute.sh '{"in":["exports/"],"out":"/tmp/checkup","hashNames":true}'
|
|
12
|
+
#
|
|
13
|
+
# Output: one JSON line (per-file source, kind, rows, date range, unmapped
|
|
14
|
+
# columns, notes) and, with --out, <name>.json per file + summary.json.
|
|
15
|
+
# Exit 0 = every file loaded, 2 = some file could not be read, 1 = usage.
|
|
16
|
+
# =============================================================================
|
|
17
|
+
set -euo pipefail
|
|
18
|
+
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
19
|
+
INGEST="${SCRIPT_DIR}/ingest.mjs"
|
|
20
|
+
|
|
21
|
+
command -v node >/dev/null 2>&1 || { printf '{"success":false,"error":"node is required but not on PATH"}\n'; exit 1; }
|
|
22
|
+
|
|
23
|
+
ARGS=()
|
|
24
|
+
if [[ $# -gt 0 && ${1:0:1} == '{' ]]; then
|
|
25
|
+
command -v jq >/dev/null 2>&1 || { printf '{"success":false,"error":"jq is required to read JSON input"}\n'; exit 1; }
|
|
26
|
+
while IFS= read -r p; do [ -n "$p" ] && ARGS+=(--in "$p"); done < <(printf '%s' "$1" | jq -r '(.in // .inputs // []) | if type=="string" then [.] else . end | .[]')
|
|
27
|
+
out=$(printf '%s' "$1" | jq -r '.out // empty'); [ -n "$out" ] && ARGS+=(--out "$out")
|
|
28
|
+
src=$(printf '%s' "$1" | jq -r '.source // empty'); [ -n "$src" ] && ARGS+=(--source "$src")
|
|
29
|
+
[ "$(printf '%s' "$1" | jq -r '.hashNames // false')" = "true" ] && ARGS+=(--hash-names)
|
|
30
|
+
[ "$(printf '%s' "$1" | jq -r '.detectOnly // false')" = "true" ] && ARGS+=(--detect-only)
|
|
31
|
+
shift
|
|
32
|
+
fi
|
|
33
|
+
ARGS+=("$@")
|
|
34
|
+
|
|
35
|
+
exec node "$INGEST" ${ARGS[@]+"${ARGS[@]}"}
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
#!/bin/bash
|
|
2
|
+
# Tests for ingest-exports — run with: bash execute.test.sh
|
|
3
|
+
# Offline: fixture CSVs only. Needs node (and jq for the JSON-input case).
|
|
4
|
+
set -u
|
|
5
|
+
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
6
|
+
EXEC="$HERE/execute.sh"
|
|
7
|
+
FX="$HERE/fixtures"
|
|
8
|
+
PASS=0; FAIL=0
|
|
9
|
+
check() {
|
|
10
|
+
local name="$1"; local got="$2"; local want="$3"
|
|
11
|
+
if [ "$got" = "$want" ]; then PASS=$((PASS+1)); else FAIL=$((FAIL+1)); echo "FAIL: $name"; echo " want: $want"; echo " got: $got"; fi
|
|
12
|
+
}
|
|
13
|
+
T="$(mktemp -d)"; trap 'rm -rf "$T"' EXIT
|
|
14
|
+
|
|
15
|
+
# 1. Every fixture is recognised as the right tool and kind
|
|
16
|
+
OUT="$(bash "$EXEC" --in "$FX" --out "$T/all")"; RC=$?
|
|
17
|
+
check "folder ingest exits 0" "$RC" "0"
|
|
18
|
+
check "7 files loaded" "$(printf '%s' "$OUT" | jq -r '.totals.files')" "7"
|
|
19
|
+
for spec in "square-transactions.csv square payment 4" "square-customers.csv square contact 3" "quickbooks-invoices.csv quickbooks invoice 3" \
|
|
20
|
+
"mailchimp-audience.csv mailchimp contact 2" "jotform-submissions.csv jotform booking 3" "jackrabbit-enrollments.csv jackrabbit booking 3" \
|
|
21
|
+
"unknown-report.csv unknown other 3"; do
|
|
22
|
+
set -- $spec
|
|
23
|
+
check "$1 source" "$(printf '%s' "$OUT" | jq -r --arg f "$1" '.files[] | select(.file==$f) | .source')" "$2"
|
|
24
|
+
check "$1 kind" "$(printf '%s' "$OUT" | jq -r --arg f "$1" '.files[] | select(.file==$f) | .kind')" "$3"
|
|
25
|
+
check "$1 rows" "$(printf '%s' "$OUT" | jq -r --arg f "$1" '.files[] | select(.file==$f) | .rows')" "$4"
|
|
26
|
+
check "$1 output file exists" "$([ -f "$T/all/${1%.csv}.json" ] && echo yes || echo no)" "yes"
|
|
27
|
+
done
|
|
28
|
+
check "summary.json written" "$([ -f "$T/all/summary.json" ] && echo yes || echo no)" "yes"
|
|
29
|
+
check "totals.rows" "$(printf '%s' "$OUT" | jq -r '.totals.rows')" "21"
|
|
30
|
+
|
|
31
|
+
# 2. Date ranges and value normalisation
|
|
32
|
+
check "square date range" "$(printf '%s' "$OUT" | jq -c '.files[] | select(.file=="square-transactions.csv") | .dateRange')" '{"from":"2026-09-02","to":"2026-09-20"}'
|
|
33
|
+
check "quickbooks US dates → ISO" "$(jq -r '.records[0].at' "$T/all/quickbooks-invoices.json")" "2026-09-01"
|
|
34
|
+
check "quickbooks amount with thousands" "$(jq -r '.records[0].amount' "$T/all/quickbooks-invoices.json")" "1250"
|
|
35
|
+
check "quickbooks paid status from open balance" "$(jq -r '.records[1].status' "$T/all/quickbooks-invoices.json")" "paid"
|
|
36
|
+
check "quickbooks due date" "$(jq -r '.records[2].dueAt' "$T/all/quickbooks-invoices.json")" "2026-10-15"
|
|
37
|
+
check "square time joined to date" "$(jq -r '.records[0].at' "$T/all/square-transactions.json")" "2026-09-02T10:14:03"
|
|
38
|
+
check "square total collected" "$(jq -r '.records[1].amount' "$T/all/square-transactions.json")" "117.45"
|
|
39
|
+
check "square refunded status" "$(jq -r '.records[2].status' "$T/all/square-transactions.json")" "refunded"
|
|
40
|
+
check "square customer name joined" "$(jq -r '.records[0].name' "$T/all/square-customers.json")" "Maria Lopez"
|
|
41
|
+
check "mailchimp tags split" "$(jq -c '.records[0].tags' "$T/all/mailchimp-audience.json")" '["parent","pottery"]'
|
|
42
|
+
check "jotform parent name" "$(jq -r '.records[0].name' "$T/all/jotform-submissions.json")" "Alex Rivera"
|
|
43
|
+
check "jotform class as item" "$(jq -r '.records[0].item' "$T/all/jotform-submissions.json")" "Kids Pottery"
|
|
44
|
+
check "jackrabbit dropped status" "$(jq -r '.records[2].status' "$T/all/jackrabbit-enrollments.json")" "dropped"
|
|
45
|
+
check "jackrabbit enrolled status" "$(jq -r '.records[0].status' "$T/all/jackrabbit-enrollments.json")" "enrolled"
|
|
46
|
+
check "jackrabbit end date" "$(jq -r '.records[2].endAt' "$T/all/jackrabbit-enrollments.json")" "2026-09-22"
|
|
47
|
+
|
|
48
|
+
# 3. Unknown CSV: original columns kept, note says could not map, quoted field intact
|
|
49
|
+
check "unknown note" "$(printf '%s' "$OUT" | jq -r '.files[] | select(.file=="unknown-report.csv") | .notes[0]' | cut -c1-13)" "could not map"
|
|
50
|
+
check "unknown keeps original columns" "$(jq -c '.records[0].original' "$T/all/unknown-report.json")" '{"Widget":"Spinner","Colour":"Blue","Units":"12","Notes":"Back-ordered, \"rush\""}'
|
|
51
|
+
check "unknown has no mapped name" "$(jq -r '.records[0] | has("name")' "$T/all/unknown-report.json")" "false"
|
|
52
|
+
check "unknown unmapped columns = all" "$(printf '%s' "$OUT" | jq -c '.files[] | select(.file=="unknown-report.csv") | .unmappedColumns')" '["Widget","Colour","Units","Notes"]'
|
|
53
|
+
|
|
54
|
+
# 4. --hash-names: no plain name/email/phone anywhere in the output
|
|
55
|
+
HOUT="$(bash "$EXEC" --in "$FX/jotform-submissions.csv" --in "$FX/jackrabbit-enrollments.csv" --out "$T/hashed" --hash-names)"
|
|
56
|
+
check "hashed flag in summary" "$(printf '%s' "$HOUT" | jq -r '.namesHashed')" "true"
|
|
57
|
+
check "no plain names in hashed jotform" "$(grep -c -E "Rivera|Natarajan|Baker|example\.test|555 010" "$T/hashed/jotform-submissions.json")" "0"
|
|
58
|
+
check "no plain names in hashed jackrabbit" "$(grep -c -E "Lopez|Rivera|Natarajan|example\.test" "$T/hashed/jackrabbit-enrollments.json")" "0"
|
|
59
|
+
check "hashed name has h: prefix" "$(jq -r '.records[0].name' "$T/hashed/jotform-submissions.json" | cut -c1-2)" "h:"
|
|
60
|
+
check "same person hashes the same across files" "$(jq -r '.records[0].email' "$T/hashed/jotform-submissions.json")" "$(jq -r '.records[1].email' "$T/hashed/jackrabbit-enrollments.json")"
|
|
61
|
+
check "child age kept (not personal)" "$(jq -r '.records[0].original["Child'"'"'s Age"]' "$T/hashed/jotform-submissions.json")" "7"
|
|
62
|
+
check "child name hashed" "$(jq -r '.records[0].original["Child'"'"'s Name"]' "$T/hashed/jotform-submissions.json" | cut -c1-2)" "h:"
|
|
63
|
+
check "rows unchanged by hashing" "$(printf '%s' "$HOUT" | jq -r '.totals.rows')" "6"
|
|
64
|
+
|
|
65
|
+
# 5. --detect-only writes nothing; JSON input form; missing file → exit 2
|
|
66
|
+
DOUT="$(bash "$EXEC" --in "$FX/square-customers.csv" --detect-only)"
|
|
67
|
+
check "detect-only source" "$(printf '%s' "$DOUT" | jq -r '.files[0].source')" "square"
|
|
68
|
+
check "detect-only has no out" "$(printf '%s' "$DOUT" | jq -r '.out')" "null"
|
|
69
|
+
JOUT="$(bash "$EXEC" "{\"in\":[\"$FX/mailchimp-audience.csv\"],\"out\":\"$T/json\",\"hashNames\":true}")"
|
|
70
|
+
check "json input works" "$(printf '%s' "$JOUT" | jq -r '.files[0].kind + " " + (.namesHashed|tostring)')" "contact true"
|
|
71
|
+
MOUT="$(bash "$EXEC" --in "$T/nope.csv" --in "$FX/unknown-report.csv" --out "$T/miss")"; RC=$?
|
|
72
|
+
check "missing file exits 2" "$RC" "2"
|
|
73
|
+
check "missing file reported, other loaded" "$(printf '%s' "$MOUT" | jq -r '(.files[0].error) + "/" + (.totals.files|tostring)')" "not found/1"
|
|
74
|
+
UOUT="$(bash "$EXEC" --out "$T/u" 2>/dev/null)"; RC=$?
|
|
75
|
+
check "no --in is a usage error" "$RC" "1"
|
|
76
|
+
|
|
77
|
+
# 6. --source forces a known mapping only when the headers fit
|
|
78
|
+
SOUT="$(bash "$EXEC" --in "$FX/unknown-report.csv" --out "$T/forced" --source square)"
|
|
79
|
+
check "--source cannot force square onto unrelated headers" "$(printf '%s' "$SOUT" | jq -r '.files[0].source')" "unknown"
|
|
80
|
+
|
|
81
|
+
echo "ingest-exports: $PASS passed, $FAIL failed"
|
|
82
|
+
[ "$FAIL" -eq 0 ]
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
Family,Student First Name,Student Last Name,Student Birth Date,Class,Session,Enroll Date,Drop Date,Tuition Fee,Primary Contact Email
|
|
2
|
+
Lopez,Sofia,Lopez,2018-04-12,Kids Pottery Sat 10am,Fall 2026,2026-08-28,,180.00,maria@example.test
|
|
3
|
+
Rivera,Sam,Rivera,2019-02-03,Kids Pottery Sat 10am,Fall 2026,2026-09-10,,180.00,alex@example.test
|
|
4
|
+
Natarajan,Anika,Natarajan,2017-07-21,Teen Wheel Tue 4pm,Fall 2026,2026-09-01,2026-09-22,220.00,priya@example.test
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
Submission Date,Submission ID,Parent Name,Child's Name,Child's Age,Email,Phone Number,Which class?,Preferred day,Anything we should know?
|
|
2
|
+
2026-09-08 14:22:10,6012345678901234567,Alex Rivera,Sam Rivera,7,alex@example.test,+1 555 010 1003,Kids Pottery,Saturday,Allergic to peanuts
|
|
3
|
+
2026-09-09 08:05:44,6012345678901234568,Priya Natarajan,Anika Natarajan,9,priya@example.test,,Teen Wheel,Tuesday,
|
|
4
|
+
2026-09-12 19:47:02,6012345678901234569,Tom Baker,Lily Baker,5,tom@example.test,+1 555 010 1004,Parent & Me,Sunday,First time
|
|
@@ -0,0 +1,3 @@
|
|
|
1
|
+
Email Address,First Name,Last Name,Address,Phone Number,MEMBER_RATING,OPTIN_TIME,OPTIN_IP,CONFIRM_TIME,CONFIRM_IP,LATITUDE,LONGITUDE,GMTOFF,DSTOFF,TIMEZONE,CC,REGION,LAST_CHANGED,LEID,EUID,NOTES,TAGS
|
|
2
|
+
maria@example.test,Maria,Lopez,"12 Elm St, Oakland, CA 94601",+1 555 010 1001,4,2025-11-02 18:04:11,,2025-11-02 18:05:02,,,,,,,,,2026-08-01 09:00:00,100001,abc001,,"parent,pottery"
|
|
3
|
+
tom@example.test,Tom,Baker,,,2,2026-02-14 10:00:00,,2026-02-14 10:01:00,,,,,,,,,2026-02-14 10:01:00,100002,abc002,,newsletter
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
Date,Transaction Type,Num,Customer,Memo/Description,Due Date,Amount,Open Balance
|
|
2
|
+
09/01/2026,Invoice,1042,Harbor Dental,September cleaning contract,09/30/2026,"1,250.00","1,250.00"
|
|
3
|
+
09/03/2026,Invoice,1043,Oak Street Cafe,Menu design,10/03/2026,380.00,0.00
|
|
4
|
+
09/15/2026,Invoice,1044,Greenline Movers,Website copy,10/15/2026,"2,140.50","2,140.50"
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
First Name,Last Name,Company Name,Email Address,Phone Number,Street Address 1,City,State,Postal Code,Creation Source,First Visit,Last Visit,Transaction Count,Total Spend,Reference ID
|
|
2
|
+
Maria,Lopez,,maria@example.test,+1 555 010 1001,12 Elm St,Oakland,CA,94601,Directory,2025-11-02,2026-09-11,6,$284.00,
|
|
3
|
+
Daniel,Chen,Chen Family,daniel@example.test,+1 555 010 1002,,Oakland,CA,94602,Instant Profile,2026-09-05,2026-09-05,1,$117.45,
|
|
4
|
+
Priya,Natarajan,,priya@example.test,,,Berkeley,CA,94704,Directory,2026-03-14,2026-08-30,3,$135.00,REF-77
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
Date,Time,Time Zone,Gross Sales,Discounts,Net Sales,Tax,Tip,Total Collected,Payment ID,Card Brand,PAN Suffix,Customer Name,Customer Email,Description,Transaction ID,Payment Status
|
|
2
|
+
2026-09-02,10:14:03,America/Los_Angeles,$45.00,$0.00,$45.00,$3.94,$5.00,$53.94,PAY-1001,Visa,4242,Maria Lopez,maria@example.test,"Kids Pottery, Saturday",TX-9001,Completed
|
|
3
|
+
2026-09-05,16:40:11,America/Los_Angeles,$120.00,$12.00,$108.00,$9.45,$0.00,$117.45,PAY-1002,Mastercard,1881,Daniel Chen,daniel@example.test,"Birthday Party Deposit",TX-9002,Completed
|
|
4
|
+
2026-09-11,09:02:45,America/Los_Angeles,$45.00,$0.00,$45.00,$3.94,$0.00,$48.94,PAY-1003,Visa,4242,Maria Lopez,maria@example.test,"Kids Pottery, Saturday",TX-9003,Refunded
|
|
5
|
+
2026-09-20,13:25:00,America/Los_Angeles,$30.00,$0.00,$30.00,$2.63,$2.00,$34.63,PAY-1004,Amex,0005,,,"Walk-in clay bar",TX-9004,Completed
|