memory-anywhere 0.0.0-stage → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +79 -2
- package/bin/memory-anywhere.mjs +59615 -0
- package/deploy/app.mjs +972 -0
- package/deploy/cdk.json +8 -0
- package/deploy/host/firstboot.sh +43 -0
- package/deploy/host/ma-updater.service +10 -0
- package/deploy/host/ma-updater.sh +252 -0
- package/deploy/host/ma-updater.timer +10 -0
- package/deploy/host/pins.env +24 -0
- package/deploy/host/release/first-store.sh +32 -0
- package/deploy/host/release/install.sh +249 -0
- package/deploy/host/release/libexec/ma-maint.sh +49 -0
- package/deploy/host/release/libexec/ma-selfcheck.sh +14 -0
- package/deploy/host/release/libexec/manifest-at.sh +42 -0
- package/deploy/host/release/libexec/outbox.sh +83 -0
- package/deploy/host/release/libexec/secrets.sh +15 -0
- package/deploy/host/release/tmpfiles.conf +13 -0
- package/deploy/host/release/units/caddy.service +26 -0
- package/deploy/host/release/units/litestream.service +28 -0
- package/deploy/host/release/units/ma-maint.service +7 -0
- package/deploy/host/release/units/ma-selfcheck.path +9 -0
- package/deploy/host/release/units/ma-selfcheck.service +9 -0
- package/deploy/host/release/units/memory-anywhere-outbox.path +9 -0
- package/deploy/host/release/units/memory-anywhere-outbox.service +21 -0
- package/deploy/host/release/units/memory-anywhere-secrets.service +25 -0
- package/deploy/host/release/units/memory-anywhere.service +35 -0
- package/deploy/release-keys/release.pub +2 -0
- package/package.json +28 -4
- package/plugins/claude-code/.claude-plugin/icon.png +0 -0
- package/plugins/claude-code/.claude-plugin/marketplace.json +12 -0
- package/plugins/claude-code/.claude-plugin/plugin.json +6 -0
- package/plugins/claude-code/.mcp.json +9 -0
- package/plugins/claude-code/README.md +11 -0
- package/plugins/claude-code/dist/memory-anywhere.mjs +59615 -0
- package/plugins/claude-code/hooks/hooks.json +30 -0
- package/plugins/claude-code/skills/memory-anywhere/SKILL.md +11 -0
- package/plugins/cursor/.cursor-plugin/plugin.json +6 -0
- package/plugins/cursor/dist/memory-anywhere.mjs +59615 -0
- package/plugins/cursor/rules/memory-anywhere.mdc +10 -0
package/deploy/cdk.json
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
#!/bin/bash
|
|
2
|
+
# First boot (deployment-aws.md §4), installed by user data as /usr/local/sbin/ma-firstboot.
|
|
3
|
+
# Idempotent. It prepares only what the updater needs to verify a release:
|
|
4
|
+
# swap, minisign (pinned hash) and the updater timer. The updater then installs
|
|
5
|
+
# the first signed release, whose host/install.sh sets up everything else.
|
|
6
|
+
set -euo pipefail
|
|
7
|
+
exec >>/var/log/ma-firstboot.log 2>&1
|
|
8
|
+
echo "== ma-firstboot $(date -u +%FT%TZ)"
|
|
9
|
+
|
|
10
|
+
# Both files are written by user data (lib/user-data.ts); pins.env is host/pins.env.
|
|
11
|
+
# shellcheck source=/dev/null
|
|
12
|
+
. /etc/memory-anywhere/stack.env
|
|
13
|
+
# shellcheck source=/dev/null
|
|
14
|
+
. /etc/memory-anywhere/pins.env
|
|
15
|
+
|
|
16
|
+
# 1 GiB swap file (§2). The server and Litestream units set MemorySwapMax=0.
|
|
17
|
+
if ! swapon --show=NAME --noheadings | grep -qx /swapfile; then
|
|
18
|
+
if [ ! -f /swapfile ]; then
|
|
19
|
+
dd if=/dev/zero of=/swapfile bs=1M count=1024 status=none
|
|
20
|
+
chmod 0600 /swapfile
|
|
21
|
+
mkswap /swapfile >/dev/null
|
|
22
|
+
fi
|
|
23
|
+
swapon /swapfile
|
|
24
|
+
grep -q '^/swapfile ' /etc/fstab || echo '/swapfile none swap sw 0 0' >>/etc/fstab
|
|
25
|
+
fi
|
|
26
|
+
|
|
27
|
+
# minisign from the pinned release archive, checked against pins.env.
|
|
28
|
+
if ! /usr/local/bin/minisign -v 2>/dev/null | grep -q "$MINISIGN_VERSION"; then
|
|
29
|
+
work=$(mktemp -d)
|
|
30
|
+
trap 'rm -rf "$work"' EXIT
|
|
31
|
+
curl -fsSL --proto '=https' --retry 5 -o "$work/minisign.tar.gz" "$MINISIGN_URL"
|
|
32
|
+
echo "$MINISIGN_SHA256 $work/minisign.tar.gz" | sha256sum -c --quiet -
|
|
33
|
+
tar -xzf "$work/minisign.tar.gz" -C "$work" minisign-linux/aarch64/minisign
|
|
34
|
+
install -m 0755 "$work/minisign-linux/aarch64/minisign" /usr/local/bin/minisign
|
|
35
|
+
fi
|
|
36
|
+
|
|
37
|
+
install -d -m 0755 /opt/memory-anywhere/releases /var/lib/memory-anywhere-updater
|
|
38
|
+
systemctl daemon-reload
|
|
39
|
+
systemctl enable --now ma-updater.timer
|
|
40
|
+
# First run now rather than in two minutes; the release may not be uploaded yet,
|
|
41
|
+
# in which case the updater exits quietly and the timer retries.
|
|
42
|
+
systemctl start --no-block ma-updater.service
|
|
43
|
+
echo "== ma-firstboot done"
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
[Unit]
|
|
2
|
+
Description=memory-anywhere updater (deployment-aws.md 7.3)
|
|
3
|
+
After=network-online.target
|
|
4
|
+
Wants=network-online.target
|
|
5
|
+
|
|
6
|
+
[Service]
|
|
7
|
+
Type=oneshot
|
|
8
|
+
ExecStart=/usr/local/sbin/ma-updater
|
|
9
|
+
# Root: it installs releases and restarts units. It has no EC2 permissions.
|
|
10
|
+
TimeoutStartSec=15min
|
|
@@ -0,0 +1,252 @@
|
|
|
1
|
+
#!/bin/bash
|
|
2
|
+
# memory-anywhere updater (deployment-aws.md §7.3), installed by user data as
|
|
3
|
+
# /usr/local/sbin/ma-updater and run as root by ma-updater.timer every 2 minutes.
|
|
4
|
+
#
|
|
5
|
+
# A release is s3://$MA_RELEASES_BUCKET/releases/<version>.tar.gz plus
|
|
6
|
+
# <version>.tar.gz.minisig; <prefix>/release/desired holds {"version","sha256"}.
|
|
7
|
+
# The tarball contains:
|
|
8
|
+
# VERSION the version string, nothing else
|
|
9
|
+
# bin/memory-anywhere-server backup --out <file> | migrate | start ...
|
|
10
|
+
# host/install.sh <release dir> root, idempotent: system packages, users, units, configs
|
|
11
|
+
# Nothing in a release runs before its SHA-256 and a trusted minisign signature
|
|
12
|
+
# (/etc/memory-anywhere/release-keys/*.pub, written by user data) both verify.
|
|
13
|
+
set -euo pipefail
|
|
14
|
+
umask 022
|
|
15
|
+
|
|
16
|
+
# shellcheck source=/dev/null
|
|
17
|
+
. /etc/memory-anywhere/stack.env
|
|
18
|
+
|
|
19
|
+
STATE=/var/lib/memory-anywhere-updater
|
|
20
|
+
RELEASES=/opt/memory-anywhere/releases
|
|
21
|
+
CURRENT=/opt/memory-anywhere/current
|
|
22
|
+
DATA=/var/lib/memory-anywhere
|
|
23
|
+
SOCKET=/run/memory-anywhere/http.sock
|
|
24
|
+
KEYS=/etc/memory-anywhere/release-keys
|
|
25
|
+
LOCK=/run/ma-updater.lock
|
|
26
|
+
|
|
27
|
+
log() { echo "ma-updater: $*"; }
|
|
28
|
+
|
|
29
|
+
aws_() { aws --region "$MA_REGION" "$@"; }
|
|
30
|
+
|
|
31
|
+
alert() { # alert <id> <message>; subject and short text only, never content
|
|
32
|
+
aws_ sns publish --topic-arn "$MA_TOPIC_ARN" --subject "memory-anywhere $MA_STAGE: $1" --message "$2" >/dev/null || log "could not publish alert $1"
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
json_field() { # json_field <field> (reads JSON on stdin)
|
|
36
|
+
python3 -c 'import json,sys; v=json.load(sys.stdin).get(sys.argv[1]); print("" if v is None else v)' "$1"
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
site_host() { # the stage's {host}: the name Caddy serves the mcp surface on
|
|
40
|
+
case "$MA_STAGE" in
|
|
41
|
+
prod) echo "$MA_HOST" ;;
|
|
42
|
+
spike) echo "spike.$MA_HOST" ;;
|
|
43
|
+
*) echo "eval.$MA_HOST" ;;
|
|
44
|
+
esac
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
health_ok() { # health_ok <version>: /v1/health over the socket reports that version
|
|
48
|
+
local body
|
|
49
|
+
# The server answers only requests that arrive as Caddy forwards them (http/front.ts): the
|
|
50
|
+
# stage's Host, X-MA-Origin and a client address. Anything else is refused as misrouted.
|
|
51
|
+
body=$(curl -fsS --max-time 5 --unix-socket "$SOCKET" -H "Host: $(site_host)" -H 'X-MA-Origin: mcp' \
|
|
52
|
+
-H 'X-Forwarded-For: 127.0.0.1' http://localhost/v1/health 2>/dev/null) || return 1
|
|
53
|
+
[ "$(printf '%s' "$body" | json_field version)" = "$1" ]
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
wait_healthy() { # wait_healthy <version>: up to 60 s
|
|
57
|
+
for _ in $(seq 1 30); do
|
|
58
|
+
if health_ok "$1"; then return 0; fi
|
|
59
|
+
sleep 2
|
|
60
|
+
done
|
|
61
|
+
return 1
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
switch_current() { # switch_current <release dir>: atomic symlink swap
|
|
65
|
+
ln -sfn "$1" "$CURRENT.new"
|
|
66
|
+
mv -T "$CURRENT.new" "$CURRENT"
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
# install_release <version> <sha256>. Called as a condition, so set -e does not apply
|
|
70
|
+
# inside it: every step checks its own result and returns 1 (refuse) before anything
|
|
71
|
+
# is switched.
|
|
72
|
+
install_release() {
|
|
73
|
+
local version=$1 sha=$2 work dest previous="" verified=""
|
|
74
|
+
work=$(mktemp -d /var/tmp/ma-release.XXXXXX)
|
|
75
|
+
# shellcheck disable=SC2064 # expand $work now: it is local to this function
|
|
76
|
+
trap "rm -rf '$work'" RETURN
|
|
77
|
+
|
|
78
|
+
if ! aws_ s3 cp --only-show-errors "s3://$MA_RELEASES_BUCKET/releases/$version.tar.gz" "$work/release.tar.gz" ||
|
|
79
|
+
! aws_ s3 cp --only-show-errors "s3://$MA_RELEASES_BUCKET/releases/$version.tar.gz.minisig" "$work/release.tar.gz.minisig"; then
|
|
80
|
+
log "could not download release $version; will retry"
|
|
81
|
+
return 1
|
|
82
|
+
fi
|
|
83
|
+
if ! echo "$sha $work/release.tar.gz" | sha256sum -c --quiet - >/dev/null 2>&1; then
|
|
84
|
+
log "refusing $version: SHA-256 does not match release/desired"
|
|
85
|
+
alert update_failed "Release $version refused: SHA-256 mismatch."
|
|
86
|
+
return 1
|
|
87
|
+
fi
|
|
88
|
+
for key in "$KEYS"/*.pub; do
|
|
89
|
+
if /usr/local/bin/minisign -V -q -p "$key" -m "$work/release.tar.gz" -x "$work/release.tar.gz.minisig"; then
|
|
90
|
+
verified=$(basename "$key")
|
|
91
|
+
break
|
|
92
|
+
fi
|
|
93
|
+
done
|
|
94
|
+
if [ -z "$verified" ]; then
|
|
95
|
+
log "refusing $version: no trusted key verifies the signature"
|
|
96
|
+
alert update_failed "Release $version refused: signature not from a key this stage trusts."
|
|
97
|
+
return 1
|
|
98
|
+
fi
|
|
99
|
+
log "release $version verified with $verified"
|
|
100
|
+
|
|
101
|
+
if [ -L "$CURRENT" ]; then previous=$(readlink -f "$CURRENT"); fi
|
|
102
|
+
if [ -n "$previous" ] && [ -f "$DATA/store.db" ]; then
|
|
103
|
+
# An earlier, failed attempt at this version left its backup; the store has moved on since.
|
|
104
|
+
rm -f "$DATA/pre-$version.db"
|
|
105
|
+
runuser -u memory-anywhere -- "$CURRENT/bin/memory-anywhere-server" backup --config /etc/memory-anywhere/server.toml --out "$DATA/pre-$version.db" ||
|
|
106
|
+
{ log "pre-update backup failed; not updating"; return 1; }
|
|
107
|
+
fi
|
|
108
|
+
|
|
109
|
+
dest="$RELEASES/$version"
|
|
110
|
+
rm -rf "$dest"
|
|
111
|
+
install -d -m 0755 "$dest"
|
|
112
|
+
if ! tar -xzf "$work/release.tar.gz" -C "$dest" --no-same-owner --no-same-permissions; then
|
|
113
|
+
log "could not unpack release $version"
|
|
114
|
+
rm -rf "$dest"
|
|
115
|
+
return 1
|
|
116
|
+
fi
|
|
117
|
+
if [ "$(tr -d '[:space:]' <"$dest/VERSION" 2>/dev/null)" != "$version" ]; then
|
|
118
|
+
log "refusing $version: VERSION file says $(cat "$dest/VERSION")"
|
|
119
|
+
rm -rf "$dest"
|
|
120
|
+
return 1
|
|
121
|
+
fi
|
|
122
|
+
if ! bash "$dest/host/install.sh" "$dest"; then
|
|
123
|
+
log "host/install.sh of $version failed; the running release is unchanged"
|
|
124
|
+
alert update_failed "Release $version: host install failed; still running the previous release."
|
|
125
|
+
return 1
|
|
126
|
+
fi
|
|
127
|
+
|
|
128
|
+
systemctl stop memory-anywhere.service 2>/dev/null || true
|
|
129
|
+
# A new instance: restore the backup or create the store (the release's host/first-store.sh).
|
|
130
|
+
if [ ! -f "$DATA/store.db" ] && ! bash "$dest/host/first-store.sh" "$dest"; then
|
|
131
|
+
log "could not create or restore the store with $version"
|
|
132
|
+
alert update_failed "Release $version: the first store could not be created or restored."
|
|
133
|
+
return 1
|
|
134
|
+
fi
|
|
135
|
+
if ! runuser -u memory-anywhere -- "$dest/bin/memory-anywhere-server" migrate --config /etc/memory-anywhere/server.toml; then
|
|
136
|
+
log "migrate failed for $version; rolling back"
|
|
137
|
+
else
|
|
138
|
+
switch_current "$dest"
|
|
139
|
+
systemctl start memory-anywhere.service || true
|
|
140
|
+
# Replication starts once a store exists (its unit has ConditionPathExists).
|
|
141
|
+
systemctl start litestream.service || true
|
|
142
|
+
fi
|
|
143
|
+
|
|
144
|
+
if wait_healthy "$version"; then
|
|
145
|
+
log "release $version healthy"
|
|
146
|
+
date -u +%s >"$STATE/healthy-since"
|
|
147
|
+
return 0
|
|
148
|
+
fi
|
|
149
|
+
|
|
150
|
+
log "release $version failed its health check; rolling back"
|
|
151
|
+
systemctl stop memory-anywhere.service || true
|
|
152
|
+
if [ -n "$previous" ]; then
|
|
153
|
+
if [ -f "$DATA/pre-$version.db" ]; then
|
|
154
|
+
install -m 0660 -o memory-anywhere -g ma-data "$DATA/pre-$version.db" "$DATA/store.db"
|
|
155
|
+
fi
|
|
156
|
+
switch_current "$previous"
|
|
157
|
+
systemctl start memory-anywhere.service || true
|
|
158
|
+
fi
|
|
159
|
+
rm -f "$STATE/healthy-since"
|
|
160
|
+
alert update_failed "Release $version failed its health check and was rolled back."
|
|
161
|
+
return 1
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
dns_check() { # hourly: {host} names resolve (DoH, port 443) to this instance's public IP
|
|
165
|
+
local stamp="$STATE/dns-checked" now names doh token ip name got
|
|
166
|
+
now=$(date -u +%s)
|
|
167
|
+
if [ -f "$stamp" ] && [ $((now - $(cat "$stamp"))) -lt 3600 ]; then return 0; fi
|
|
168
|
+
echo "$now" >"$stamp"
|
|
169
|
+
case "$MA_STAGE" in
|
|
170
|
+
spike) names="spike.$MA_HOST" ;;
|
|
171
|
+
*) names="$(site_host) admin.$(site_host)" ;;
|
|
172
|
+
esac
|
|
173
|
+
doh=https://cloudflare-dns.com/dns-query
|
|
174
|
+
if [ -f /etc/memory-anywhere/server.toml ]; then
|
|
175
|
+
doh=$(sed -n 's/^[[:space:]]*doh_url[[:space:]]*=[[:space:]]*"\(https:[^"]*\)".*/\1/p' /etc/memory-anywhere/server.toml | head -n1)
|
|
176
|
+
doh=${doh:-https://cloudflare-dns.com/dns-query}
|
|
177
|
+
fi
|
|
178
|
+
token=$(curl -fsS -X PUT -H 'X-aws-ec2-metadata-token-ttl-seconds: 60' http://169.254.169.254/latest/api/token)
|
|
179
|
+
ip=$(curl -fsS -H "X-aws-ec2-metadata-token: $token" http://169.254.169.254/latest/meta-data/public-ipv4)
|
|
180
|
+
for name in $names; do
|
|
181
|
+
got=$(curl -fsS --max-time 10 -H 'accept: application/dns-json' "$doh?name=$name&type=A" |
|
|
182
|
+
python3 -c 'import json,sys; print(" ".join(sorted(a["data"] for a in json.load(sys.stdin).get("Answer",[]) if a.get("type")==1)))') || got="(lookup failed)"
|
|
183
|
+
if [ "$got" != "$ip" ]; then
|
|
184
|
+
log "dns_mismatch: $name resolves to '$got', expected $ip"
|
|
185
|
+
alert dns_mismatch "$name resolves to '$got', not this instance's Elastic IP $ip."
|
|
186
|
+
fi
|
|
187
|
+
done
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
acme_account() { # store Caddy's ACME account URI once it exists (§4 step 6)
|
|
191
|
+
local stamp="$STATE/acme-account-uri" file uri
|
|
192
|
+
file=$(find /var/lib/caddy -path '*acme-v02.api.letsencrypt.org-directory/users/*' -name '*.json' 2>/dev/null | head -n1)
|
|
193
|
+
[ -n "$file" ] || return 0
|
|
194
|
+
uri=$(json_field location <"$file") || return 0
|
|
195
|
+
[ -n "$uri" ] || return 0
|
|
196
|
+
if [ ! -f "$stamp" ] || [ "$(cat "$stamp")" != "$uri" ]; then
|
|
197
|
+
aws_ ssm put-parameter --name "$MA_PREFIX/acme/account-uri" --type String --overwrite --value "$uri" >/dev/null
|
|
198
|
+
echo "$uri" >"$stamp"
|
|
199
|
+
log "stored ACME account URI"
|
|
200
|
+
fi
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
housekeeping() { # §7.3 step 8
|
|
204
|
+
local since now keep
|
|
205
|
+
if [ -f "$STATE/healthy-since" ]; then
|
|
206
|
+
since=$(cat "$STATE/healthy-since")
|
|
207
|
+
now=$(date -u +%s)
|
|
208
|
+
if [ $((now - since)) -ge 86400 ]; then
|
|
209
|
+
find "$DATA" -maxdepth 1 -type f \( -name 'pre-*.db' -o -name 'pre-migrate-*.db' -o -name 'pre-restore-*.db' \) -mmin +1440 \
|
|
210
|
+
-exec shred -n0 -z -u {} +
|
|
211
|
+
fi
|
|
212
|
+
fi
|
|
213
|
+
# Keep the current release and the two before it.
|
|
214
|
+
keep=$(readlink -f "$CURRENT" 2>/dev/null || true)
|
|
215
|
+
find "$RELEASES" -mindepth 1 -maxdepth 1 -type d -printf '%T@ %p\n' | sort -rn | awk '{print $2}' | tail -n +4 |
|
|
216
|
+
while read -r dir; do
|
|
217
|
+
if [ "$dir" != "$keep" ]; then rm -rf "$dir"; fi
|
|
218
|
+
done
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
main() {
|
|
222
|
+
exec 9>"$LOCK"
|
|
223
|
+
flock -n 9 || { log "another run is in progress"; exit 0; }
|
|
224
|
+
install -d -m 0700 "$STATE"
|
|
225
|
+
|
|
226
|
+
local desired version sha current=""
|
|
227
|
+
if ! desired=$(aws_ ssm get-parameter --name "$MA_PREFIX/release/desired" --query Parameter.Value --output text 2>/dev/null); then
|
|
228
|
+
log "no release/desired yet"
|
|
229
|
+
exit 0
|
|
230
|
+
fi
|
|
231
|
+
version=$(printf '%s' "$desired" | json_field version)
|
|
232
|
+
sha=$(printf '%s' "$desired" | json_field sha256)
|
|
233
|
+
if ! [[ "$version" =~ ^[0-9A-Za-z][0-9A-Za-z.+_-]{0,63}$ ]] || ! [[ "$sha" =~ ^[0-9a-f]{64}$ ]]; then
|
|
234
|
+
log "release/desired is malformed; ignoring"
|
|
235
|
+
exit 1
|
|
236
|
+
fi
|
|
237
|
+
if [ -f "$CURRENT/VERSION" ]; then current=$(tr -d '[:space:]' <"$CURRENT/VERSION"); fi
|
|
238
|
+
|
|
239
|
+
if [ "$version" != "$current" ]; then
|
|
240
|
+
install_release "$version" "$sha" || true
|
|
241
|
+
elif ! health_ok "$version"; then
|
|
242
|
+
rm -f "$STATE/healthy-since"
|
|
243
|
+
elif [ ! -f "$STATE/healthy-since" ]; then
|
|
244
|
+
date -u +%s >"$STATE/healthy-since"
|
|
245
|
+
fi
|
|
246
|
+
|
|
247
|
+
dns_check || log "dns check failed to run"
|
|
248
|
+
acme_account || log "acme account step failed"
|
|
249
|
+
housekeeping
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
main "$@"
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# Pinned instance binaries (deployment-aws.md §4 step 2, auth-and-security.md §10).
|
|
2
|
+
# Every download on the instance is checked against these before it is used.
|
|
3
|
+
# Sources, checked 2026-11-07:
|
|
4
|
+
# Node https://nodejs.org/dist/v24.22.2/SHASUMS256.txt
|
|
5
|
+
# Caddy caddy_2.11.9_checksums.txt (SHA-512) from the GitHub release
|
|
6
|
+
# Litestream checksums.txt from the v0.5.18 GitHub release
|
|
7
|
+
# minisign 0.12 Linux tarball verified with the author's minisign key
|
|
8
|
+
# RWQf6LRCGA9i53mlYecO4IzT51TGPpvWucNSCh1CBM0QTaLn73Y7GFO3, then hashed
|
|
9
|
+
|
|
10
|
+
NODE_VERSION=24.22.2
|
|
11
|
+
NODE_URL=https://nodejs.org/dist/v24.22.2/node-v24.22.2-linux-arm64.tar.xz
|
|
12
|
+
NODE_SHA256=eb187837a2230d4047161ac0be008584e16b46f726e9c9490ee35dc6b17d9ba8
|
|
13
|
+
|
|
14
|
+
CADDY_VERSION=2.11.9
|
|
15
|
+
CADDY_URL=https://github.com/caddyserver/caddy/releases/download/v2.11.9/caddy_2.11.9_linux_arm64.tar.gz
|
|
16
|
+
CADDY_SHA512=9e5e76d7888542924de2c25353ae87c5f77e2a22c5b1198633cae13688922282e05383d142599205c88d8d80eee5e967d291214d7abce4835329a191b20dd6e0
|
|
17
|
+
|
|
18
|
+
LITESTREAM_VERSION=0.5.18
|
|
19
|
+
LITESTREAM_URL=https://github.com/benbjohnson/litestream/releases/download/v0.5.18/litestream-0.5.18-linux-arm64.tar.gz
|
|
20
|
+
LITESTREAM_SHA256=922a09e6a7ac67f2095571894dc6382e94e1dd4cdd9674883ddf1a8e74ab982d
|
|
21
|
+
|
|
22
|
+
MINISIGN_VERSION=0.12
|
|
23
|
+
MINISIGN_URL=https://github.com/jedisct1/minisign/releases/download/0.12/minisign-0.12-linux.tar.gz
|
|
24
|
+
MINISIGN_SHA256=9a599b48ba6eb7b1e80f12f36b94ceca7c00b7a5173c95c3efc88d9822957e73
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
#!/bin/bash
|
|
2
|
+
# host/first-store.sh <release dir>: run as root by ma-updater when the instance has no
|
|
3
|
+
# store.db (deployment-aws.md §4 step 8). If the backup bucket holds a replica, Litestream
|
|
4
|
+
# restores it and the server starts in restored state (§5.5); otherwise `init` creates a new
|
|
5
|
+
# store. `init` also downloads and checks the embedding model, so it runs in both cases.
|
|
6
|
+
set -euo pipefail
|
|
7
|
+
|
|
8
|
+
REL=${1:?usage: first-store.sh <release dir>}
|
|
9
|
+
DATA=/var/lib/memory-anywhere
|
|
10
|
+
CONFIG=/etc/memory-anywhere/server.toml
|
|
11
|
+
log() { echo "first-store.sh: $*"; }
|
|
12
|
+
|
|
13
|
+
# The manifest the restored chain must reach is fetched first: if the bucket cannot be read, the
|
|
14
|
+
# instance stops here, before a store exists that the next updater run would start unchecked.
|
|
15
|
+
install -d -m 0770 -o memory-anywhere -g ma-data "$DATA/maint"
|
|
16
|
+
bash "$REL/host/libexec/manifest-at.sh" "" "$DATA/maint/restored-manifest.json"
|
|
17
|
+
if runuser -u litestream -- /usr/local/bin/litestream restore -config /etc/litestream.yml -if-db-not-exists -if-replica-exists "$DATA/store.db" &&
|
|
18
|
+
[ -f "$DATA/store.db" ]; then
|
|
19
|
+
chown memory-anywhere:ma-data "$DATA/store.db"
|
|
20
|
+
chmod 0660 "$DATA/store.db"
|
|
21
|
+
printf '{"reason":"lost","to":""}\n' >"$DATA/maint/restored"
|
|
22
|
+
chown memory-anywhere:ma-data "$DATA/maint/restored"
|
|
23
|
+
chmod 0660 "$DATA/maint/restored"
|
|
24
|
+
# The restore drill compares this with the hash `ma-operator.sh pause` printed on the old instance.
|
|
25
|
+
hash=$(runuser -u memory-anywhere -- "$REL/bin/memory-anywhere-server" dump --canonical --config "$CONFIG" | sha256sum | cut -d' ' -f1)
|
|
26
|
+
printf '%s %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$hash" >"$DATA/maint/restored-store-hash"
|
|
27
|
+
log "restored the store from the backup (store hash $hash); the server starts in restored state"
|
|
28
|
+
else
|
|
29
|
+
rm -f "$DATA/maint/restored-manifest.json"
|
|
30
|
+
fi
|
|
31
|
+
runuser -u memory-anywhere -- "$REL/bin/memory-anywhere-server" init --config "$CONFIG" >/dev/null
|
|
32
|
+
log "store ready"
|
|
@@ -0,0 +1,249 @@
|
|
|
1
|
+
#!/bin/bash
|
|
2
|
+
# host/install.sh <release dir>: run as root by ma-updater after the release's
|
|
3
|
+
# signature and SHA-256 verified (deployment-aws.md §4, §7.3). Idempotent: every
|
|
4
|
+
# step checks what is already there. It never starts the server; the updater does.
|
|
5
|
+
set -euo pipefail
|
|
6
|
+
umask 022
|
|
7
|
+
|
|
8
|
+
REL=${1:?usage: install.sh <release dir>}
|
|
9
|
+
HOSTDIR="$REL/host"
|
|
10
|
+
# shellcheck source=/dev/null
|
|
11
|
+
. /etc/memory-anywhere/stack.env
|
|
12
|
+
# shellcheck source=/dev/null
|
|
13
|
+
. /etc/memory-anywhere/pins.env
|
|
14
|
+
|
|
15
|
+
log() { echo "install.sh: $*"; }
|
|
16
|
+
|
|
17
|
+
fetch_verified() { # fetch_verified <url> <algo:sha256|sha512> <hash> <out>
|
|
18
|
+
curl -fsSL --proto '=https' --retry 5 -o "$4" "$1"
|
|
19
|
+
echo "$3 $4" | "$2sum" -c --quiet -
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
# ---- 1. users and groups (§4 step 1, step 7) ----
|
|
23
|
+
ensure_group() { getent group "$1" >/dev/null || groupadd --system "$1"; }
|
|
24
|
+
ensure_user() { # ensure_user <name> <home>
|
|
25
|
+
id -u "$1" >/dev/null 2>&1 || useradd --system --home-dir "$2" --no-create-home --shell /sbin/nologin --user-group "$1"
|
|
26
|
+
}
|
|
27
|
+
ensure_user memory-anywhere /var/lib/memory-anywhere
|
|
28
|
+
ensure_user litestream /var/lib/litestream
|
|
29
|
+
ensure_user ma-aws /var/lib/ma-aws
|
|
30
|
+
ensure_user caddy /var/lib/caddy
|
|
31
|
+
ensure_group ma-data # members: memory-anywhere, litestream only
|
|
32
|
+
usermod -a -G ma-data memory-anywhere
|
|
33
|
+
usermod -a -G ma-data litestream
|
|
34
|
+
|
|
35
|
+
# ---- 2. pinned binaries (§4 step 2) ----
|
|
36
|
+
work=$(mktemp -d /var/tmp/ma-install.XXXXXX)
|
|
37
|
+
trap 'rm -rf "$work"' EXIT
|
|
38
|
+
|
|
39
|
+
if [ "$(cat /opt/memory-anywhere/node/VERSION 2>/dev/null || true)" != "$NODE_VERSION" ]; then
|
|
40
|
+
log "installing Node $NODE_VERSION"
|
|
41
|
+
fetch_verified "$NODE_URL" sha256 "$NODE_SHA256" "$work/node.tar.xz"
|
|
42
|
+
dest="/opt/memory-anywhere/node-$NODE_VERSION"
|
|
43
|
+
rm -rf "$dest" && install -d "$dest"
|
|
44
|
+
tar -xJf "$work/node.tar.xz" -C "$dest" --strip-components=1 --no-same-owner
|
|
45
|
+
echo "$NODE_VERSION" >"$dest/VERSION"
|
|
46
|
+
ln -sfn "$dest" /opt/memory-anywhere/node
|
|
47
|
+
fi
|
|
48
|
+
|
|
49
|
+
if ! /usr/local/bin/caddy version 2>/dev/null | grep -q "^v$CADDY_VERSION "; then
|
|
50
|
+
log "installing Caddy $CADDY_VERSION"
|
|
51
|
+
fetch_verified "$CADDY_URL" sha512 "$CADDY_SHA512" "$work/caddy.tar.gz"
|
|
52
|
+
tar -xzf "$work/caddy.tar.gz" -C "$work" caddy
|
|
53
|
+
install -m 0755 "$work/caddy" /usr/local/bin/caddy
|
|
54
|
+
fi
|
|
55
|
+
|
|
56
|
+
if ! /usr/local/bin/litestream version 2>/dev/null | grep -q "$LITESTREAM_VERSION"; then
|
|
57
|
+
log "installing Litestream $LITESTREAM_VERSION"
|
|
58
|
+
fetch_verified "$LITESTREAM_URL" sha256 "$LITESTREAM_SHA256" "$work/litestream.tar.gz"
|
|
59
|
+
tar -xzf "$work/litestream.tar.gz" -C "$work" litestream
|
|
60
|
+
install -m 0755 "$work/litestream" /usr/local/bin/litestream
|
|
61
|
+
fi
|
|
62
|
+
|
|
63
|
+
# ---- 3. IMDS firewall (§4 step 3): the server and Caddy users cannot reach IMDS ----
|
|
64
|
+
if ! command -v nft >/dev/null; then dnf install -y -q nftables; fi
|
|
65
|
+
install -d -m 0755 /etc/nftables
|
|
66
|
+
# Numeric UIDs: the users exist by now, and a set of numbers needs no name quoting.
|
|
67
|
+
cat >/etc/nftables/memory-anywhere.nft <<EOF
|
|
68
|
+
# memory-anywhere and caddy cannot reach IMDS (auth-and-security.md §9).
|
|
69
|
+
table inet memory_anywhere {
|
|
70
|
+
chain output {
|
|
71
|
+
type filter hook output priority 0; policy accept;
|
|
72
|
+
ip daddr 169.254.169.254 meta skuid { $(id -u memory-anywhere), $(id -u caddy) } reject
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
EOF
|
|
76
|
+
conf=/etc/sysconfig/nftables.conf
|
|
77
|
+
touch "$conf"
|
|
78
|
+
grep -q 'include "/etc/nftables/memory-anywhere.nft"' "$conf" || echo 'include "/etc/nftables/memory-anywhere.nft"' >>"$conf"
|
|
79
|
+
nft list table inet memory_anywhere >/dev/null 2>&1 && nft delete table inet memory_anywhere
|
|
80
|
+
nft -f /etc/nftables/memory-anywhere.nft
|
|
81
|
+
systemctl enable --quiet nftables.service
|
|
82
|
+
|
|
83
|
+
# ---- 4. directories (§4 step 7) ----
|
|
84
|
+
# setgid: files the server creates (store.db, WAL) belong to ma-data, so Litestream can use them.
|
|
85
|
+
install -d -m 2770 -o memory-anywhere -g ma-data /var/lib/memory-anywhere
|
|
86
|
+
install -d -m 2770 -o memory-anywhere -g ma-data /var/lib/memory-anywhere/maint
|
|
87
|
+
# The store and its WAL are shared by the server and Litestream: group ma-data, 0660. Release
|
|
88
|
+
# 0.2.0 created them 0640, which neither could write past; repair such files on upgrade.
|
|
89
|
+
for f in /var/lib/memory-anywhere/store.db /var/lib/memory-anywhere/store.db-wal /var/lib/memory-anywhere/store.db-shm; do
|
|
90
|
+
if [ -e "$f" ]; then chgrp ma-data "$f" && chmod 0660 "$f"; fi
|
|
91
|
+
done
|
|
92
|
+
install -d -m 0755 /etc/memory-anywhere /usr/local/libexec/memory-anywhere
|
|
93
|
+
install -m 0644 "$HOSTDIR/tmpfiles.conf" /etc/tmpfiles.d/memory-anywhere.conf
|
|
94
|
+
systemd-tmpfiles --create /etc/tmpfiles.d/memory-anywhere.conf
|
|
95
|
+
|
|
96
|
+
# ---- 5. configuration ----
|
|
97
|
+
case "$MA_STAGE" in
|
|
98
|
+
prod) site=$MA_HOST ;;
|
|
99
|
+
spike) site="spike.$MA_HOST" ;;
|
|
100
|
+
*) site="eval.$MA_HOST" ;;
|
|
101
|
+
esac
|
|
102
|
+
# The owner's time zone (stack parameter OwnerTimezone); CloudFormation already checked
|
|
103
|
+
# its shape, and this check keeps anything else out of the generated files.
|
|
104
|
+
tz=${MA_OWNER_TZ:-UTC}
|
|
105
|
+
if ! [[ "$tz" =~ ^[A-Za-z][A-Za-z0-9_+-]*(/[A-Za-z0-9_+-]+){0,2}$ ]] || [ ! -e "/usr/share/zoneinfo/$tz" ]; then
|
|
106
|
+
log "unknown time zone '$tz'; using UTC"
|
|
107
|
+
tz=UTC
|
|
108
|
+
fi
|
|
109
|
+
cat >/etc/memory-anywhere/server.toml <<EOF
|
|
110
|
+
# Written by host/install.sh from /etc/memory-anywhere/stack.env (config-and-cli.md §5).
|
|
111
|
+
[server]
|
|
112
|
+
mode = "aws"
|
|
113
|
+
public_url = "https://$site"
|
|
114
|
+
listen = "unix:/run/memory-anywhere/http.sock"
|
|
115
|
+
data_dir = "/var/lib/memory-anywhere"
|
|
116
|
+
models_dir = "/var/lib/memory-anywhere/models"
|
|
117
|
+
|
|
118
|
+
[owner]
|
|
119
|
+
timezone = "$tz"
|
|
120
|
+
|
|
121
|
+
[llm]
|
|
122
|
+
provider = "anthropic"
|
|
123
|
+
|
|
124
|
+
[alerts]
|
|
125
|
+
sns_topic_arn = "$MA_TOPIC_ARN"
|
|
126
|
+
|
|
127
|
+
[deployment]
|
|
128
|
+
doh_url = "https://cloudflare-dns.com/dns-query"
|
|
129
|
+
# Eval stacks only: the owner-authenticated selfcheck routes.
|
|
130
|
+
selfcheck = $([[ "$MA_STAGE" == eval-* ]] && echo true || echo false)
|
|
131
|
+
EOF
|
|
132
|
+
chmod 0644 /etc/memory-anywhere/server.toml
|
|
133
|
+
|
|
134
|
+
install -d -m 0755 /etc/caddy
|
|
135
|
+
{
|
|
136
|
+
cat <<EOF
|
|
137
|
+
{
|
|
138
|
+
admin off
|
|
139
|
+
email $MA_ALERT_EMAIL
|
|
140
|
+
# Port 80 stays closed: no HTTP redirect listener and no HTTP-01 challenge;
|
|
141
|
+
# Let's Encrypt is the only issuer, matching the CAA record (§4 step 4, §6.2).
|
|
142
|
+
auto_https disable_redirects
|
|
143
|
+
cert_issuer acme {
|
|
144
|
+
disable_http_challenge
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
EOF
|
|
148
|
+
site_block() { # site_block <name> <origin label>
|
|
149
|
+
cat <<EOF
|
|
150
|
+
|
|
151
|
+
$1 {
|
|
152
|
+
reverse_proxy unix//run/memory-anywhere/http.sock {
|
|
153
|
+
header_up X-Forwarded-For {remote_host}
|
|
154
|
+
header_up X-MA-Origin $2
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
EOF
|
|
158
|
+
}
|
|
159
|
+
case "$MA_STAGE" in
|
|
160
|
+
spike) site_block "spike.$MA_HOST" mcp ;; # the spike serves everything on one name (§9)
|
|
161
|
+
prod) site_block "$MA_HOST" mcp && site_block "admin.$MA_HOST" admin ;;
|
|
162
|
+
*) site_block "eval.$MA_HOST" mcp && site_block "admin.eval.$MA_HOST" admin ;;
|
|
163
|
+
esac
|
|
164
|
+
} >/etc/caddy/Caddyfile
|
|
165
|
+
chmod 0644 /etc/caddy/Caddyfile
|
|
166
|
+
/usr/local/bin/caddy validate --config /etc/caddy/Caddyfile --adapter caddyfile >/dev/null
|
|
167
|
+
|
|
168
|
+
cat >/etc/litestream.yml <<EOF
|
|
169
|
+
# deployment-aws.md §5.1 (v0.5 layout). No keys: the SDK uses the instance role.
|
|
170
|
+
snapshot:
|
|
171
|
+
interval: 24h
|
|
172
|
+
retention: 168h
|
|
173
|
+
|
|
174
|
+
dbs:
|
|
175
|
+
- path: /var/lib/memory-anywhere/store.db
|
|
176
|
+
replica:
|
|
177
|
+
type: s3
|
|
178
|
+
bucket: $MA_BACKUP_BUCKET
|
|
179
|
+
path: litestream/store
|
|
180
|
+
region: $MA_REGION
|
|
181
|
+
sse-kms-key-id: $MA_KEY_ARN
|
|
182
|
+
sync-interval: 1s
|
|
183
|
+
meta-path: /run/litestream/store
|
|
184
|
+
EOF
|
|
185
|
+
chmod 0644 /etc/litestream.yml
|
|
186
|
+
|
|
187
|
+
# ---- 5b. log shipping (§2): the CloudWatch agent reads files, not the journal ----
|
|
188
|
+
# From Amazon Linux's own signed repository, whose release is fixed by the pinned AMI.
|
|
189
|
+
rpm -q amazon-cloudwatch-agent >/dev/null 2>&1 || dnf install -y -q amazon-cloudwatch-agent
|
|
190
|
+
install -d -m 0750 -o root -g root /var/log/memory-anywhere
|
|
191
|
+
cat >/etc/logrotate.d/memory-anywhere <<EOF
|
|
192
|
+
/var/log/memory-anywhere/server.log {
|
|
193
|
+
daily
|
|
194
|
+
rotate 7
|
|
195
|
+
missingok
|
|
196
|
+
notifempty
|
|
197
|
+
compress
|
|
198
|
+
delaycompress
|
|
199
|
+
copytruncate
|
|
200
|
+
}
|
|
201
|
+
EOF
|
|
202
|
+
cwa=/opt/aws/amazon-cloudwatch-agent
|
|
203
|
+
cat >"$cwa/etc/memory-anywhere.json" <<EOF
|
|
204
|
+
{
|
|
205
|
+
"agent": { "run_as_user": "root" },
|
|
206
|
+
"logs": {
|
|
207
|
+
"logs_collected": {
|
|
208
|
+
"files": {
|
|
209
|
+
"collect_list": [
|
|
210
|
+
{ "file_path": "/var/log/memory-anywhere/server.log", "log_group_name": "$MA_LOG_GROUP", "log_stream_name": "{instance_id}", "retention_in_days": -1 }
|
|
211
|
+
]
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
EOF
|
|
217
|
+
"$cwa/bin/amazon-cloudwatch-agent-ctl" -a fetch-config -m ec2 -s -c "file:$cwa/etc/memory-anywhere.json" >/dev/null
|
|
218
|
+
|
|
219
|
+
# ---- 6. helpers and units (§4 step 5) ----
|
|
220
|
+
install -m 0755 "$HOSTDIR/libexec/secrets.sh" /usr/local/libexec/memory-anywhere/secrets.sh
|
|
221
|
+
install -m 0755 "$HOSTDIR/libexec/outbox.sh" /usr/local/libexec/memory-anywhere/outbox.sh
|
|
222
|
+
install -m 0755 "$HOSTDIR/libexec/manifest-at.sh" /usr/local/libexec/memory-anywhere/manifest-at.sh
|
|
223
|
+
install -m 0755 "$HOSTDIR/libexec/ma-maint.sh" /usr/local/libexec/memory-anywhere/ma-maint.sh
|
|
224
|
+
install -m 0755 "$HOSTDIR/libexec/ma-selfcheck.sh" /usr/local/libexec/memory-anywhere/ma-selfcheck.sh
|
|
225
|
+
for unit in "$HOSTDIR"/units/*; do install -m 0644 "$unit" "/etc/systemd/system/$(basename "$unit")"; done
|
|
226
|
+
# The maintenance timer runs at 04:15 in the owner's time zone, so it is written here.
|
|
227
|
+
cat >/etc/systemd/system/ma-maint.timer <<EOF
|
|
228
|
+
[Unit]
|
|
229
|
+
Description=memory-anywhere nightly maintenance at 04:15 owner time (deployment-aws.md §4)
|
|
230
|
+
|
|
231
|
+
[Timer]
|
|
232
|
+
OnCalendar=*-*-* 04:15:00 $tz
|
|
233
|
+
Persistent=true
|
|
234
|
+
|
|
235
|
+
[Install]
|
|
236
|
+
WantedBy=timers.target
|
|
237
|
+
EOF
|
|
238
|
+
chmod 0755 "$REL/bin/memory-anywhere-server"
|
|
239
|
+
systemctl daemon-reload
|
|
240
|
+
systemctl enable --quiet memory-anywhere-secrets.service memory-anywhere.service memory-anywhere-outbox.path caddy.service litestream.service ma-maint.timer
|
|
241
|
+
systemctl start memory-anywhere-outbox.path ma-maint.timer
|
|
242
|
+
# The on-instance checks of suite S run on eval stacks only.
|
|
243
|
+
if [[ "$MA_STAGE" == eval-* ]]; then
|
|
244
|
+
systemctl enable --quiet --now ma-selfcheck.path
|
|
245
|
+
else
|
|
246
|
+
systemctl disable --quiet --now ma-selfcheck.path 2>/dev/null || true
|
|
247
|
+
fi
|
|
248
|
+
systemctl restart caddy.service
|
|
249
|
+
log "host configured for $MA_STAGE ($site)"
|