memory-anywhere 0.0.0-stage → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +79 -2
  3. package/bin/memory-anywhere.mjs +59615 -0
  4. package/deploy/app.mjs +972 -0
  5. package/deploy/cdk.json +8 -0
  6. package/deploy/host/firstboot.sh +43 -0
  7. package/deploy/host/ma-updater.service +10 -0
  8. package/deploy/host/ma-updater.sh +252 -0
  9. package/deploy/host/ma-updater.timer +10 -0
  10. package/deploy/host/pins.env +24 -0
  11. package/deploy/host/release/first-store.sh +32 -0
  12. package/deploy/host/release/install.sh +249 -0
  13. package/deploy/host/release/libexec/ma-maint.sh +49 -0
  14. package/deploy/host/release/libexec/ma-selfcheck.sh +14 -0
  15. package/deploy/host/release/libexec/manifest-at.sh +42 -0
  16. package/deploy/host/release/libexec/outbox.sh +83 -0
  17. package/deploy/host/release/libexec/secrets.sh +15 -0
  18. package/deploy/host/release/tmpfiles.conf +13 -0
  19. package/deploy/host/release/units/caddy.service +26 -0
  20. package/deploy/host/release/units/litestream.service +28 -0
  21. package/deploy/host/release/units/ma-maint.service +7 -0
  22. package/deploy/host/release/units/ma-selfcheck.path +9 -0
  23. package/deploy/host/release/units/ma-selfcheck.service +9 -0
  24. package/deploy/host/release/units/memory-anywhere-outbox.path +9 -0
  25. package/deploy/host/release/units/memory-anywhere-outbox.service +21 -0
  26. package/deploy/host/release/units/memory-anywhere-secrets.service +25 -0
  27. package/deploy/host/release/units/memory-anywhere.service +35 -0
  28. package/deploy/release-keys/release.pub +2 -0
  29. package/package.json +28 -4
  30. package/plugins/claude-code/.claude-plugin/icon.png +0 -0
  31. package/plugins/claude-code/.claude-plugin/marketplace.json +12 -0
  32. package/plugins/claude-code/.claude-plugin/plugin.json +6 -0
  33. package/plugins/claude-code/.mcp.json +9 -0
  34. package/plugins/claude-code/README.md +11 -0
  35. package/plugins/claude-code/dist/memory-anywhere.mjs +59615 -0
  36. package/plugins/claude-code/hooks/hooks.json +30 -0
  37. package/plugins/claude-code/skills/memory-anywhere/SKILL.md +11 -0
  38. package/plugins/cursor/.cursor-plugin/plugin.json +6 -0
  39. package/plugins/cursor/dist/memory-anywhere.mjs +59615 -0
  40. package/plugins/cursor/rules/memory-anywhere.mdc +10 -0
@@ -0,0 +1,8 @@
1
+ {
2
+ "app": "node bin/app.ts",
3
+ "output": "cdk.out",
4
+ "context": {
5
+ "@aws-cdk/aws-ec2:restrictDefaultSecurityGroup": false,
6
+ "@aws-cdk/core:validateAgainstDefaultRules": true
7
+ }
8
+ }
@@ -0,0 +1,43 @@
1
+ #!/bin/bash
2
+ # First boot (deployment-aws.md §4), installed by user data as /usr/local/sbin/ma-firstboot.
3
+ # Idempotent. It prepares only what the updater needs to verify a release:
4
+ # swap, minisign (pinned hash) and the updater timer. The updater then installs
5
+ # the first signed release, whose host/install.sh sets up everything else.
6
+ set -euo pipefail
7
+ exec >>/var/log/ma-firstboot.log 2>&1
8
+ echo "== ma-firstboot $(date -u +%FT%TZ)"
9
+
10
+ # Both files are written by user data (lib/user-data.ts); pins.env is host/pins.env.
11
+ # shellcheck source=/dev/null
12
+ . /etc/memory-anywhere/stack.env
13
+ # shellcheck source=/dev/null
14
+ . /etc/memory-anywhere/pins.env
15
+
16
+ # 1 GiB swap file (§2). The server and Litestream units set MemorySwapMax=0.
17
+ if ! swapon --show=NAME --noheadings | grep -qx /swapfile; then
18
+ if [ ! -f /swapfile ]; then
19
+ dd if=/dev/zero of=/swapfile bs=1M count=1024 status=none
20
+ chmod 0600 /swapfile
21
+ mkswap /swapfile >/dev/null
22
+ fi
23
+ swapon /swapfile
24
+ grep -q '^/swapfile ' /etc/fstab || echo '/swapfile none swap sw 0 0' >>/etc/fstab
25
+ fi
26
+
27
+ # minisign from the pinned release archive, checked against pins.env.
28
+ if ! /usr/local/bin/minisign -v 2>/dev/null | grep -q "$MINISIGN_VERSION"; then
29
+ work=$(mktemp -d)
30
+ trap 'rm -rf "$work"' EXIT
31
+ curl -fsSL --proto '=https' --retry 5 -o "$work/minisign.tar.gz" "$MINISIGN_URL"
32
+ echo "$MINISIGN_SHA256 $work/minisign.tar.gz" | sha256sum -c --quiet -
33
+ tar -xzf "$work/minisign.tar.gz" -C "$work" minisign-linux/aarch64/minisign
34
+ install -m 0755 "$work/minisign-linux/aarch64/minisign" /usr/local/bin/minisign
35
+ fi
36
+
37
+ install -d -m 0755 /opt/memory-anywhere/releases /var/lib/memory-anywhere-updater
38
+ systemctl daemon-reload
39
+ systemctl enable --now ma-updater.timer
40
+ # First run now rather than in two minutes; the release may not be uploaded yet,
41
+ # in which case the updater exits quietly and the timer retries.
42
+ systemctl start --no-block ma-updater.service
43
+ echo "== ma-firstboot done"
@@ -0,0 +1,10 @@
1
+ [Unit]
2
+ Description=memory-anywhere updater (deployment-aws.md 7.3)
3
+ After=network-online.target
4
+ Wants=network-online.target
5
+
6
+ [Service]
7
+ Type=oneshot
8
+ ExecStart=/usr/local/sbin/ma-updater
9
+ # Root: it installs releases and restarts units. It has no EC2 permissions.
10
+ TimeoutStartSec=15min
@@ -0,0 +1,252 @@
1
+ #!/bin/bash
2
+ # memory-anywhere updater (deployment-aws.md §7.3), installed by user data as
3
+ # /usr/local/sbin/ma-updater and run as root by ma-updater.timer every 2 minutes.
4
+ #
5
+ # A release is s3://$MA_RELEASES_BUCKET/releases/<version>.tar.gz plus
6
+ # <version>.tar.gz.minisig; <prefix>/release/desired holds {"version","sha256"}.
7
+ # The tarball contains:
8
+ # VERSION the version string, nothing else
9
+ # bin/memory-anywhere-server backup --out <file> | migrate | start ...
10
+ # host/install.sh <release dir> root, idempotent: system packages, users, units, configs
11
+ # Nothing in a release runs before its SHA-256 and a trusted minisign signature
12
+ # (/etc/memory-anywhere/release-keys/*.pub, written by user data) both verify.
13
+ set -euo pipefail
14
+ umask 022
15
+
16
+ # shellcheck source=/dev/null
17
+ . /etc/memory-anywhere/stack.env
18
+
19
+ STATE=/var/lib/memory-anywhere-updater
20
+ RELEASES=/opt/memory-anywhere/releases
21
+ CURRENT=/opt/memory-anywhere/current
22
+ DATA=/var/lib/memory-anywhere
23
+ SOCKET=/run/memory-anywhere/http.sock
24
+ KEYS=/etc/memory-anywhere/release-keys
25
+ LOCK=/run/ma-updater.lock
26
+
27
+ log() { echo "ma-updater: $*"; }
28
+
29
+ aws_() { aws --region "$MA_REGION" "$@"; }
30
+
31
+ alert() { # alert <id> <message>; subject and short text only, never content
32
+ aws_ sns publish --topic-arn "$MA_TOPIC_ARN" --subject "memory-anywhere $MA_STAGE: $1" --message "$2" >/dev/null || log "could not publish alert $1"
33
+ }
34
+
35
+ json_field() { # json_field <field> (reads JSON on stdin)
36
+ python3 -c 'import json,sys; v=json.load(sys.stdin).get(sys.argv[1]); print("" if v is None else v)' "$1"
37
+ }
38
+
39
+ site_host() { # the stage's {host}: the name Caddy serves the mcp surface on
40
+ case "$MA_STAGE" in
41
+ prod) echo "$MA_HOST" ;;
42
+ spike) echo "spike.$MA_HOST" ;;
43
+ *) echo "eval.$MA_HOST" ;;
44
+ esac
45
+ }
46
+
47
+ health_ok() { # health_ok <version>: /v1/health over the socket reports that version
48
+ local body
49
+ # The server answers only requests that arrive as Caddy forwards them (http/front.ts): the
50
+ # stage's Host, X-MA-Origin and a client address. Anything else is refused as misrouted.
51
+ body=$(curl -fsS --max-time 5 --unix-socket "$SOCKET" -H "Host: $(site_host)" -H 'X-MA-Origin: mcp' \
52
+ -H 'X-Forwarded-For: 127.0.0.1' http://localhost/v1/health 2>/dev/null) || return 1
53
+ [ "$(printf '%s' "$body" | json_field version)" = "$1" ]
54
+ }
55
+
56
+ wait_healthy() { # wait_healthy <version>: up to 60 s
57
+ for _ in $(seq 1 30); do
58
+ if health_ok "$1"; then return 0; fi
59
+ sleep 2
60
+ done
61
+ return 1
62
+ }
63
+
64
+ switch_current() { # switch_current <release dir>: atomic symlink swap
65
+ ln -sfn "$1" "$CURRENT.new"
66
+ mv -T "$CURRENT.new" "$CURRENT"
67
+ }
68
+
69
+ # install_release <version> <sha256>. Called as a condition, so set -e does not apply
70
+ # inside it: every step checks its own result and returns 1 (refuse) before anything
71
+ # is switched.
72
+ install_release() {
73
+ local version=$1 sha=$2 work dest previous="" verified=""
74
+ work=$(mktemp -d /var/tmp/ma-release.XXXXXX)
75
+ # shellcheck disable=SC2064 # expand $work now: it is local to this function
76
+ trap "rm -rf '$work'" RETURN
77
+
78
+ if ! aws_ s3 cp --only-show-errors "s3://$MA_RELEASES_BUCKET/releases/$version.tar.gz" "$work/release.tar.gz" ||
79
+ ! aws_ s3 cp --only-show-errors "s3://$MA_RELEASES_BUCKET/releases/$version.tar.gz.minisig" "$work/release.tar.gz.minisig"; then
80
+ log "could not download release $version; will retry"
81
+ return 1
82
+ fi
83
+ if ! echo "$sha $work/release.tar.gz" | sha256sum -c --quiet - >/dev/null 2>&1; then
84
+ log "refusing $version: SHA-256 does not match release/desired"
85
+ alert update_failed "Release $version refused: SHA-256 mismatch."
86
+ return 1
87
+ fi
88
+ for key in "$KEYS"/*.pub; do
89
+ if /usr/local/bin/minisign -V -q -p "$key" -m "$work/release.tar.gz" -x "$work/release.tar.gz.minisig"; then
90
+ verified=$(basename "$key")
91
+ break
92
+ fi
93
+ done
94
+ if [ -z "$verified" ]; then
95
+ log "refusing $version: no trusted key verifies the signature"
96
+ alert update_failed "Release $version refused: signature not from a key this stage trusts."
97
+ return 1
98
+ fi
99
+ log "release $version verified with $verified"
100
+
101
+ if [ -L "$CURRENT" ]; then previous=$(readlink -f "$CURRENT"); fi
102
+ if [ -n "$previous" ] && [ -f "$DATA/store.db" ]; then
103
+ # An earlier, failed attempt at this version left its backup; the store has moved on since.
104
+ rm -f "$DATA/pre-$version.db"
105
+ runuser -u memory-anywhere -- "$CURRENT/bin/memory-anywhere-server" backup --config /etc/memory-anywhere/server.toml --out "$DATA/pre-$version.db" ||
106
+ { log "pre-update backup failed; not updating"; return 1; }
107
+ fi
108
+
109
+ dest="$RELEASES/$version"
110
+ rm -rf "$dest"
111
+ install -d -m 0755 "$dest"
112
+ if ! tar -xzf "$work/release.tar.gz" -C "$dest" --no-same-owner --no-same-permissions; then
113
+ log "could not unpack release $version"
114
+ rm -rf "$dest"
115
+ return 1
116
+ fi
117
+ if [ "$(tr -d '[:space:]' <"$dest/VERSION" 2>/dev/null)" != "$version" ]; then
118
+ log "refusing $version: VERSION file says $(cat "$dest/VERSION")"
119
+ rm -rf "$dest"
120
+ return 1
121
+ fi
122
+ if ! bash "$dest/host/install.sh" "$dest"; then
123
+ log "host/install.sh of $version failed; the running release is unchanged"
124
+ alert update_failed "Release $version: host install failed; still running the previous release."
125
+ return 1
126
+ fi
127
+
128
+ systemctl stop memory-anywhere.service 2>/dev/null || true
129
+ # A new instance: restore the backup or create the store (the release's host/first-store.sh).
130
+ if [ ! -f "$DATA/store.db" ] && ! bash "$dest/host/first-store.sh" "$dest"; then
131
+ log "could not create or restore the store with $version"
132
+ alert update_failed "Release $version: the first store could not be created or restored."
133
+ return 1
134
+ fi
135
+ if ! runuser -u memory-anywhere -- "$dest/bin/memory-anywhere-server" migrate --config /etc/memory-anywhere/server.toml; then
136
+ log "migrate failed for $version; rolling back"
137
+ else
138
+ switch_current "$dest"
139
+ systemctl start memory-anywhere.service || true
140
+ # Replication starts once a store exists (its unit has ConditionPathExists).
141
+ systemctl start litestream.service || true
142
+ fi
143
+
144
+ if wait_healthy "$version"; then
145
+ log "release $version healthy"
146
+ date -u +%s >"$STATE/healthy-since"
147
+ return 0
148
+ fi
149
+
150
+ log "release $version failed its health check; rolling back"
151
+ systemctl stop memory-anywhere.service || true
152
+ if [ -n "$previous" ]; then
153
+ if [ -f "$DATA/pre-$version.db" ]; then
154
+ install -m 0660 -o memory-anywhere -g ma-data "$DATA/pre-$version.db" "$DATA/store.db"
155
+ fi
156
+ switch_current "$previous"
157
+ systemctl start memory-anywhere.service || true
158
+ fi
159
+ rm -f "$STATE/healthy-since"
160
+ alert update_failed "Release $version failed its health check and was rolled back."
161
+ return 1
162
+ }
163
+
164
+ dns_check() { # hourly: {host} names resolve (DoH, port 443) to this instance's public IP
165
+ local stamp="$STATE/dns-checked" now names doh token ip name got
166
+ now=$(date -u +%s)
167
+ if [ -f "$stamp" ] && [ $((now - $(cat "$stamp"))) -lt 3600 ]; then return 0; fi
168
+ echo "$now" >"$stamp"
169
+ case "$MA_STAGE" in
170
+ spike) names="spike.$MA_HOST" ;;
171
+ *) names="$(site_host) admin.$(site_host)" ;;
172
+ esac
173
+ doh=https://cloudflare-dns.com/dns-query
174
+ if [ -f /etc/memory-anywhere/server.toml ]; then
175
+ doh=$(sed -n 's/^[[:space:]]*doh_url[[:space:]]*=[[:space:]]*"\(https:[^"]*\)".*/\1/p' /etc/memory-anywhere/server.toml | head -n1)
176
+ doh=${doh:-https://cloudflare-dns.com/dns-query}
177
+ fi
178
+ token=$(curl -fsS -X PUT -H 'X-aws-ec2-metadata-token-ttl-seconds: 60' http://169.254.169.254/latest/api/token)
179
+ ip=$(curl -fsS -H "X-aws-ec2-metadata-token: $token" http://169.254.169.254/latest/meta-data/public-ipv4)
180
+ for name in $names; do
181
+ got=$(curl -fsS --max-time 10 -H 'accept: application/dns-json' "$doh?name=$name&type=A" |
182
+ python3 -c 'import json,sys; print(" ".join(sorted(a["data"] for a in json.load(sys.stdin).get("Answer",[]) if a.get("type")==1)))') || got="(lookup failed)"
183
+ if [ "$got" != "$ip" ]; then
184
+ log "dns_mismatch: $name resolves to '$got', expected $ip"
185
+ alert dns_mismatch "$name resolves to '$got', not this instance's Elastic IP $ip."
186
+ fi
187
+ done
188
+ }
189
+
190
+ acme_account() { # store Caddy's ACME account URI once it exists (§4 step 6)
191
+ local stamp="$STATE/acme-account-uri" file uri
192
+ file=$(find /var/lib/caddy -path '*acme-v02.api.letsencrypt.org-directory/users/*' -name '*.json' 2>/dev/null | head -n1)
193
+ [ -n "$file" ] || return 0
194
+ uri=$(json_field location <"$file") || return 0
195
+ [ -n "$uri" ] || return 0
196
+ if [ ! -f "$stamp" ] || [ "$(cat "$stamp")" != "$uri" ]; then
197
+ aws_ ssm put-parameter --name "$MA_PREFIX/acme/account-uri" --type String --overwrite --value "$uri" >/dev/null
198
+ echo "$uri" >"$stamp"
199
+ log "stored ACME account URI"
200
+ fi
201
+ }
202
+
203
+ housekeeping() { # §7.3 step 8
204
+ local since now keep
205
+ if [ -f "$STATE/healthy-since" ]; then
206
+ since=$(cat "$STATE/healthy-since")
207
+ now=$(date -u +%s)
208
+ if [ $((now - since)) -ge 86400 ]; then
209
+ find "$DATA" -maxdepth 1 -type f \( -name 'pre-*.db' -o -name 'pre-migrate-*.db' -o -name 'pre-restore-*.db' \) -mmin +1440 \
210
+ -exec shred -n0 -z -u {} +
211
+ fi
212
+ fi
213
+ # Keep the current release and the two before it.
214
+ keep=$(readlink -f "$CURRENT" 2>/dev/null || true)
215
+ find "$RELEASES" -mindepth 1 -maxdepth 1 -type d -printf '%T@ %p\n' | sort -rn | awk '{print $2}' | tail -n +4 |
216
+ while read -r dir; do
217
+ if [ "$dir" != "$keep" ]; then rm -rf "$dir"; fi
218
+ done
219
+ }
220
+
221
+ main() {
222
+ exec 9>"$LOCK"
223
+ flock -n 9 || { log "another run is in progress"; exit 0; }
224
+ install -d -m 0700 "$STATE"
225
+
226
+ local desired version sha current=""
227
+ if ! desired=$(aws_ ssm get-parameter --name "$MA_PREFIX/release/desired" --query Parameter.Value --output text 2>/dev/null); then
228
+ log "no release/desired yet"
229
+ exit 0
230
+ fi
231
+ version=$(printf '%s' "$desired" | json_field version)
232
+ sha=$(printf '%s' "$desired" | json_field sha256)
233
+ if ! [[ "$version" =~ ^[0-9A-Za-z][0-9A-Za-z.+_-]{0,63}$ ]] || ! [[ "$sha" =~ ^[0-9a-f]{64}$ ]]; then
234
+ log "release/desired is malformed; ignoring"
235
+ exit 1
236
+ fi
237
+ if [ -f "$CURRENT/VERSION" ]; then current=$(tr -d '[:space:]' <"$CURRENT/VERSION"); fi
238
+
239
+ if [ "$version" != "$current" ]; then
240
+ install_release "$version" "$sha" || true
241
+ elif ! health_ok "$version"; then
242
+ rm -f "$STATE/healthy-since"
243
+ elif [ ! -f "$STATE/healthy-since" ]; then
244
+ date -u +%s >"$STATE/healthy-since"
245
+ fi
246
+
247
+ dns_check || log "dns check failed to run"
248
+ acme_account || log "acme account step failed"
249
+ housekeeping
250
+ }
251
+
252
+ main "$@"
@@ -0,0 +1,10 @@
1
+ [Unit]
2
+ Description=Run the memory-anywhere updater every 2 minutes
3
+
4
+ [Timer]
5
+ OnBootSec=1min
6
+ OnUnitInactiveSec=2min
7
+ AccuracySec=10s
8
+
9
+ [Install]
10
+ WantedBy=timers.target
@@ -0,0 +1,24 @@
1
+ # Pinned instance binaries (deployment-aws.md §4 step 2, auth-and-security.md §10).
2
+ # Every download on the instance is checked against these before it is used.
3
+ # Sources, checked 2026-11-07:
4
+ # Node https://nodejs.org/dist/v24.22.2/SHASUMS256.txt
5
+ # Caddy caddy_2.11.9_checksums.txt (SHA-512) from the GitHub release
6
+ # Litestream checksums.txt from the v0.5.18 GitHub release
7
+ # minisign 0.12 Linux tarball verified with the author's minisign key
8
+ # RWQf6LRCGA9i53mlYecO4IzT51TGPpvWucNSCh1CBM0QTaLn73Y7GFO3, then hashed
9
+
10
+ NODE_VERSION=24.22.2
11
+ NODE_URL=https://nodejs.org/dist/v24.22.2/node-v24.22.2-linux-arm64.tar.xz
12
+ NODE_SHA256=eb187837a2230d4047161ac0be008584e16b46f726e9c9490ee35dc6b17d9ba8
13
+
14
+ CADDY_VERSION=2.11.9
15
+ CADDY_URL=https://github.com/caddyserver/caddy/releases/download/v2.11.9/caddy_2.11.9_linux_arm64.tar.gz
16
+ CADDY_SHA512=9e5e76d7888542924de2c25353ae87c5f77e2a22c5b1198633cae13688922282e05383d142599205c88d8d80eee5e967d291214d7abce4835329a191b20dd6e0
17
+
18
+ LITESTREAM_VERSION=0.5.18
19
+ LITESTREAM_URL=https://github.com/benbjohnson/litestream/releases/download/v0.5.18/litestream-0.5.18-linux-arm64.tar.gz
20
+ LITESTREAM_SHA256=922a09e6a7ac67f2095571894dc6382e94e1dd4cdd9674883ddf1a8e74ab982d
21
+
22
+ MINISIGN_VERSION=0.12
23
+ MINISIGN_URL=https://github.com/jedisct1/minisign/releases/download/0.12/minisign-0.12-linux.tar.gz
24
+ MINISIGN_SHA256=9a599b48ba6eb7b1e80f12f36b94ceca7c00b7a5173c95c3efc88d9822957e73
@@ -0,0 +1,32 @@
1
+ #!/bin/bash
2
+ # host/first-store.sh <release dir>: run as root by ma-updater when the instance has no
3
+ # store.db (deployment-aws.md §4 step 8). If the backup bucket holds a replica, Litestream
4
+ # restores it and the server starts in restored state (§5.5); otherwise `init` creates a new
5
+ # store. `init` also downloads and checks the embedding model, so it runs in both cases.
6
+ set -euo pipefail
7
+
8
+ REL=${1:?usage: first-store.sh <release dir>}
9
+ DATA=/var/lib/memory-anywhere
10
+ CONFIG=/etc/memory-anywhere/server.toml
11
+ log() { echo "first-store.sh: $*"; }
12
+
13
+ # The manifest the restored chain must reach is fetched first: if the bucket cannot be read, the
14
+ # instance stops here, before a store exists that the next updater run would start unchecked.
15
+ install -d -m 0770 -o memory-anywhere -g ma-data "$DATA/maint"
16
+ bash "$REL/host/libexec/manifest-at.sh" "" "$DATA/maint/restored-manifest.json"
17
+ if runuser -u litestream -- /usr/local/bin/litestream restore -config /etc/litestream.yml -if-db-not-exists -if-replica-exists "$DATA/store.db" &&
18
+ [ -f "$DATA/store.db" ]; then
19
+ chown memory-anywhere:ma-data "$DATA/store.db"
20
+ chmod 0660 "$DATA/store.db"
21
+ printf '{"reason":"lost","to":""}\n' >"$DATA/maint/restored"
22
+ chown memory-anywhere:ma-data "$DATA/maint/restored"
23
+ chmod 0660 "$DATA/maint/restored"
24
+ # The restore drill compares this with the hash `ma-operator.sh pause` printed on the old instance.
25
+ hash=$(runuser -u memory-anywhere -- "$REL/bin/memory-anywhere-server" dump --canonical --config "$CONFIG" | sha256sum | cut -d' ' -f1)
26
+ printf '%s %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$hash" >"$DATA/maint/restored-store-hash"
27
+ log "restored the store from the backup (store hash $hash); the server starts in restored state"
28
+ else
29
+ rm -f "$DATA/maint/restored-manifest.json"
30
+ fi
31
+ runuser -u memory-anywhere -- "$REL/bin/memory-anywhere-server" init --config "$CONFIG" >/dev/null
32
+ log "store ready"
@@ -0,0 +1,249 @@
1
+ #!/bin/bash
2
+ # host/install.sh <release dir>: run as root by ma-updater after the release's
3
+ # signature and SHA-256 verified (deployment-aws.md §4, §7.3). Idempotent: every
4
+ # step checks what is already there. It never starts the server; the updater does.
5
+ set -euo pipefail
6
+ umask 022
7
+
8
+ REL=${1:?usage: install.sh <release dir>}
9
+ HOSTDIR="$REL/host"
10
+ # shellcheck source=/dev/null
11
+ . /etc/memory-anywhere/stack.env
12
+ # shellcheck source=/dev/null
13
+ . /etc/memory-anywhere/pins.env
14
+
15
+ log() { echo "install.sh: $*"; }
16
+
17
+ fetch_verified() { # fetch_verified <url> <algo:sha256|sha512> <hash> <out>
18
+ curl -fsSL --proto '=https' --retry 5 -o "$4" "$1"
19
+ echo "$3 $4" | "$2sum" -c --quiet -
20
+ }
21
+
22
+ # ---- 1. users and groups (§4 step 1, step 7) ----
23
+ ensure_group() { getent group "$1" >/dev/null || groupadd --system "$1"; }
24
+ ensure_user() { # ensure_user <name> <home>
25
+ id -u "$1" >/dev/null 2>&1 || useradd --system --home-dir "$2" --no-create-home --shell /sbin/nologin --user-group "$1"
26
+ }
27
+ ensure_user memory-anywhere /var/lib/memory-anywhere
28
+ ensure_user litestream /var/lib/litestream
29
+ ensure_user ma-aws /var/lib/ma-aws
30
+ ensure_user caddy /var/lib/caddy
31
+ ensure_group ma-data # members: memory-anywhere, litestream only
32
+ usermod -a -G ma-data memory-anywhere
33
+ usermod -a -G ma-data litestream
34
+
35
+ # ---- 2. pinned binaries (§4 step 2) ----
36
+ work=$(mktemp -d /var/tmp/ma-install.XXXXXX)
37
+ trap 'rm -rf "$work"' EXIT
38
+
39
+ if [ "$(cat /opt/memory-anywhere/node/VERSION 2>/dev/null || true)" != "$NODE_VERSION" ]; then
40
+ log "installing Node $NODE_VERSION"
41
+ fetch_verified "$NODE_URL" sha256 "$NODE_SHA256" "$work/node.tar.xz"
42
+ dest="/opt/memory-anywhere/node-$NODE_VERSION"
43
+ rm -rf "$dest" && install -d "$dest"
44
+ tar -xJf "$work/node.tar.xz" -C "$dest" --strip-components=1 --no-same-owner
45
+ echo "$NODE_VERSION" >"$dest/VERSION"
46
+ ln -sfn "$dest" /opt/memory-anywhere/node
47
+ fi
48
+
49
+ if ! /usr/local/bin/caddy version 2>/dev/null | grep -q "^v$CADDY_VERSION "; then
50
+ log "installing Caddy $CADDY_VERSION"
51
+ fetch_verified "$CADDY_URL" sha512 "$CADDY_SHA512" "$work/caddy.tar.gz"
52
+ tar -xzf "$work/caddy.tar.gz" -C "$work" caddy
53
+ install -m 0755 "$work/caddy" /usr/local/bin/caddy
54
+ fi
55
+
56
+ if ! /usr/local/bin/litestream version 2>/dev/null | grep -q "$LITESTREAM_VERSION"; then
57
+ log "installing Litestream $LITESTREAM_VERSION"
58
+ fetch_verified "$LITESTREAM_URL" sha256 "$LITESTREAM_SHA256" "$work/litestream.tar.gz"
59
+ tar -xzf "$work/litestream.tar.gz" -C "$work" litestream
60
+ install -m 0755 "$work/litestream" /usr/local/bin/litestream
61
+ fi
62
+
63
+ # ---- 3. IMDS firewall (§4 step 3): the server and Caddy users cannot reach IMDS ----
64
+ if ! command -v nft >/dev/null; then dnf install -y -q nftables; fi
65
+ install -d -m 0755 /etc/nftables
66
+ # Numeric UIDs: the users exist by now, and a set of numbers needs no name quoting.
67
+ cat >/etc/nftables/memory-anywhere.nft <<EOF
68
+ # memory-anywhere and caddy cannot reach IMDS (auth-and-security.md §9).
69
+ table inet memory_anywhere {
70
+ chain output {
71
+ type filter hook output priority 0; policy accept;
72
+ ip daddr 169.254.169.254 meta skuid { $(id -u memory-anywhere), $(id -u caddy) } reject
73
+ }
74
+ }
75
+ EOF
76
+ conf=/etc/sysconfig/nftables.conf
77
+ touch "$conf"
78
+ grep -q 'include "/etc/nftables/memory-anywhere.nft"' "$conf" || echo 'include "/etc/nftables/memory-anywhere.nft"' >>"$conf"
79
+ nft list table inet memory_anywhere >/dev/null 2>&1 && nft delete table inet memory_anywhere
80
+ nft -f /etc/nftables/memory-anywhere.nft
81
+ systemctl enable --quiet nftables.service
82
+
83
+ # ---- 4. directories (§4 step 7) ----
84
+ # setgid: files the server creates (store.db, WAL) belong to ma-data, so Litestream can use them.
85
+ install -d -m 2770 -o memory-anywhere -g ma-data /var/lib/memory-anywhere
86
+ install -d -m 2770 -o memory-anywhere -g ma-data /var/lib/memory-anywhere/maint
87
+ # The store and its WAL are shared by the server and Litestream: group ma-data, 0660. Release
88
+ # 0.2.0 created them 0640, which neither could write past; repair such files on upgrade.
89
+ for f in /var/lib/memory-anywhere/store.db /var/lib/memory-anywhere/store.db-wal /var/lib/memory-anywhere/store.db-shm; do
90
+ if [ -e "$f" ]; then chgrp ma-data "$f" && chmod 0660 "$f"; fi
91
+ done
92
+ install -d -m 0755 /etc/memory-anywhere /usr/local/libexec/memory-anywhere
93
+ install -m 0644 "$HOSTDIR/tmpfiles.conf" /etc/tmpfiles.d/memory-anywhere.conf
94
+ systemd-tmpfiles --create /etc/tmpfiles.d/memory-anywhere.conf
95
+
96
+ # ---- 5. configuration ----
97
+ case "$MA_STAGE" in
98
+ prod) site=$MA_HOST ;;
99
+ spike) site="spike.$MA_HOST" ;;
100
+ *) site="eval.$MA_HOST" ;;
101
+ esac
102
+ # The owner's time zone (stack parameter OwnerTimezone); CloudFormation already checked
103
+ # its shape, and this check keeps anything else out of the generated files.
104
+ tz=${MA_OWNER_TZ:-UTC}
105
+ if ! [[ "$tz" =~ ^[A-Za-z][A-Za-z0-9_+-]*(/[A-Za-z0-9_+-]+){0,2}$ ]] || [ ! -e "/usr/share/zoneinfo/$tz" ]; then
106
+ log "unknown time zone '$tz'; using UTC"
107
+ tz=UTC
108
+ fi
109
+ cat >/etc/memory-anywhere/server.toml <<EOF
110
+ # Written by host/install.sh from /etc/memory-anywhere/stack.env (config-and-cli.md §5).
111
+ [server]
112
+ mode = "aws"
113
+ public_url = "https://$site"
114
+ listen = "unix:/run/memory-anywhere/http.sock"
115
+ data_dir = "/var/lib/memory-anywhere"
116
+ models_dir = "/var/lib/memory-anywhere/models"
117
+
118
+ [owner]
119
+ timezone = "$tz"
120
+
121
+ [llm]
122
+ provider = "anthropic"
123
+
124
+ [alerts]
125
+ sns_topic_arn = "$MA_TOPIC_ARN"
126
+
127
+ [deployment]
128
+ doh_url = "https://cloudflare-dns.com/dns-query"
129
+ # Eval stacks only: the owner-authenticated selfcheck routes.
130
+ selfcheck = $([[ "$MA_STAGE" == eval-* ]] && echo true || echo false)
131
+ EOF
132
+ chmod 0644 /etc/memory-anywhere/server.toml
133
+
134
+ install -d -m 0755 /etc/caddy
135
+ {
136
+ cat <<EOF
137
+ {
138
+ admin off
139
+ email $MA_ALERT_EMAIL
140
+ # Port 80 stays closed: no HTTP redirect listener and no HTTP-01 challenge;
141
+ # Let's Encrypt is the only issuer, matching the CAA record (§4 step 4, §6.2).
142
+ auto_https disable_redirects
143
+ cert_issuer acme {
144
+ disable_http_challenge
145
+ }
146
+ }
147
+ EOF
148
+ site_block() { # site_block <name> <origin label>
149
+ cat <<EOF
150
+
151
+ $1 {
152
+ reverse_proxy unix//run/memory-anywhere/http.sock {
153
+ header_up X-Forwarded-For {remote_host}
154
+ header_up X-MA-Origin $2
155
+ }
156
+ }
157
+ EOF
158
+ }
159
+ case "$MA_STAGE" in
160
+ spike) site_block "spike.$MA_HOST" mcp ;; # the spike serves everything on one name (§9)
161
+ prod) site_block "$MA_HOST" mcp && site_block "admin.$MA_HOST" admin ;;
162
+ *) site_block "eval.$MA_HOST" mcp && site_block "admin.eval.$MA_HOST" admin ;;
163
+ esac
164
+ } >/etc/caddy/Caddyfile
165
+ chmod 0644 /etc/caddy/Caddyfile
166
+ /usr/local/bin/caddy validate --config /etc/caddy/Caddyfile --adapter caddyfile >/dev/null
167
+
168
+ cat >/etc/litestream.yml <<EOF
169
+ # deployment-aws.md §5.1 (v0.5 layout). No keys: the SDK uses the instance role.
170
+ snapshot:
171
+ interval: 24h
172
+ retention: 168h
173
+
174
+ dbs:
175
+ - path: /var/lib/memory-anywhere/store.db
176
+ replica:
177
+ type: s3
178
+ bucket: $MA_BACKUP_BUCKET
179
+ path: litestream/store
180
+ region: $MA_REGION
181
+ sse-kms-key-id: $MA_KEY_ARN
182
+ sync-interval: 1s
183
+ meta-path: /run/litestream/store
184
+ EOF
185
+ chmod 0644 /etc/litestream.yml
186
+
187
+ # ---- 5b. log shipping (§2): the CloudWatch agent reads files, not the journal ----
188
+ # From Amazon Linux's own signed repository, whose release is fixed by the pinned AMI.
189
+ rpm -q amazon-cloudwatch-agent >/dev/null 2>&1 || dnf install -y -q amazon-cloudwatch-agent
190
+ install -d -m 0750 -o root -g root /var/log/memory-anywhere
191
+ cat >/etc/logrotate.d/memory-anywhere <<EOF
192
+ /var/log/memory-anywhere/server.log {
193
+ daily
194
+ rotate 7
195
+ missingok
196
+ notifempty
197
+ compress
198
+ delaycompress
199
+ copytruncate
200
+ }
201
+ EOF
202
+ cwa=/opt/aws/amazon-cloudwatch-agent
203
+ cat >"$cwa/etc/memory-anywhere.json" <<EOF
204
+ {
205
+ "agent": { "run_as_user": "root" },
206
+ "logs": {
207
+ "logs_collected": {
208
+ "files": {
209
+ "collect_list": [
210
+ { "file_path": "/var/log/memory-anywhere/server.log", "log_group_name": "$MA_LOG_GROUP", "log_stream_name": "{instance_id}", "retention_in_days": -1 }
211
+ ]
212
+ }
213
+ }
214
+ }
215
+ }
216
+ EOF
217
+ "$cwa/bin/amazon-cloudwatch-agent-ctl" -a fetch-config -m ec2 -s -c "file:$cwa/etc/memory-anywhere.json" >/dev/null
218
+
219
+ # ---- 6. helpers and units (§4 step 5) ----
220
+ install -m 0755 "$HOSTDIR/libexec/secrets.sh" /usr/local/libexec/memory-anywhere/secrets.sh
221
+ install -m 0755 "$HOSTDIR/libexec/outbox.sh" /usr/local/libexec/memory-anywhere/outbox.sh
222
+ install -m 0755 "$HOSTDIR/libexec/manifest-at.sh" /usr/local/libexec/memory-anywhere/manifest-at.sh
223
+ install -m 0755 "$HOSTDIR/libexec/ma-maint.sh" /usr/local/libexec/memory-anywhere/ma-maint.sh
224
+ install -m 0755 "$HOSTDIR/libexec/ma-selfcheck.sh" /usr/local/libexec/memory-anywhere/ma-selfcheck.sh
225
+ for unit in "$HOSTDIR"/units/*; do install -m 0644 "$unit" "/etc/systemd/system/$(basename "$unit")"; done
226
+ # The maintenance timer runs at 04:15 in the owner's time zone, so it is written here.
227
+ cat >/etc/systemd/system/ma-maint.timer <<EOF
228
+ [Unit]
229
+ Description=memory-anywhere nightly maintenance at 04:15 owner time (deployment-aws.md §4)
230
+
231
+ [Timer]
232
+ OnCalendar=*-*-* 04:15:00 $tz
233
+ Persistent=true
234
+
235
+ [Install]
236
+ WantedBy=timers.target
237
+ EOF
238
+ chmod 0755 "$REL/bin/memory-anywhere-server"
239
+ systemctl daemon-reload
240
+ systemctl enable --quiet memory-anywhere-secrets.service memory-anywhere.service memory-anywhere-outbox.path caddy.service litestream.service ma-maint.timer
241
+ systemctl start memory-anywhere-outbox.path ma-maint.timer
242
+ # The on-instance checks of suite S run on eval stacks only.
243
+ if [[ "$MA_STAGE" == eval-* ]]; then
244
+ systemctl enable --quiet --now ma-selfcheck.path
245
+ else
246
+ systemctl disable --quiet --now ma-selfcheck.path 2>/dev/null || true
247
+ fi
248
+ systemctl restart caddy.service
249
+ log "host configured for $MA_STAGE ($site)"