@vibelord/lanes 1.2.0 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. package/package.json +1 -1
  2. package/payload/lanes/README.md +4 -3
  3. package/payload/lanes/VERSION +1 -1
  4. package/payload/lanes/bin/lanes +301 -45
  5. package/payload/lanes/container/lane +42 -1
  6. package/payload/lanes/container/lane-mcp +65 -6
  7. package/payload/lanes/container/lane_core.py +178 -0
  8. package/payload/lanes/docs/ARCHITECTURE.md +192 -20
  9. package/payload/lanes/docs/DESIGN.md +6 -3
  10. package/payload/lanes/docs/INSTALL.md +25 -8
  11. package/payload/lanes/docs/RUNBOOK.md +369 -8
  12. package/payload/lanes/docs/USERGUIDE.md +201 -25
  13. package/payload/lanes/holders.lock +5 -0
  14. package/payload/lanes/install/enable-docs.sh +78 -0
  15. package/payload/lanes/install/host-install.sh +39 -5
  16. package/payload/lanes/libexec/docs_store.py +995 -0
  17. package/payload/lanes/libexec/docsd.py +2010 -0
  18. package/payload/lanes/libexec/mdlite.py +575 -0
  19. package/payload/lanes/libexec/members.py +5 -0
  20. package/payload/lanes/libexec/notify.py +65 -5
  21. package/payload/lanes/libexec/sidebar.py +106 -13
  22. package/payload/lanes/libexec/statusd.py +97 -5
  23. package/payload/lanes/libexec/tokens.py +31 -9
  24. package/payload/lanes/libexec/update.py +94 -22
  25. package/payload/lanes/libexec/uploads.py +538 -0
  26. package/payload/lanes/systemd/lanes-docsd.service +29 -0
  27. package/payload/lanes/web/Caddyfile +24 -1
  28. package/payload/lanes/web/README.md +86 -8
  29. package/payload/lanes/web/browser.js +2 -4
  30. package/payload/lanes/web/docs.css +52 -0
  31. package/payload/lanes/web/docs.js +36 -0
  32. package/payload/lanes/web/mobile.js +3 -2
  33. package/payload/lanes/web/upload.css +17 -0
  34. package/payload/lanes/web/upload.js +84 -0
  35. package/payload/manifest.json +4 -3
@@ -0,0 +1,2010 @@
1
+ #!/usr/bin/env python3
2
+ """lanes-docsd — the docs cockpit's web service (W10 W3, LNS-104).
3
+
4
+ What an agent publishes from a lane, and what a person uploads, is opened here from the phone or laptop shell over the
5
+ tailnet, bound to the person's login. `lanes-docsd.service` runs this file on 127.0.0.1:8185 (`LANES_DOCS_PORT`;
6
+ `--port`, `--bind`: a loopback address only — the service trusts the identity headers its proxy chain sets), behind the
7
+ loopback Caddy of `lanes-mobile.service` (`handle_path /lanes-docs/*`, which strips `/lanes-docs`) behind Tailscale
8
+ Serve (`https://<host>.<tailnet>.ts.net:8443/lanes-docs/...`). Standard library only. The documents, their receipts,
9
+ counts and docs.log are docs_store.py's (W1); Markdown is rendered by mdlite.py (W2).
10
+
11
+ Who is served (plan "Auth and threat model", binding). A request is answered only when ALL hold, checked in this order
12
+ by Service.gate, each refusal a 403 with its word:
13
+ not-through-serve enrold.through_serve: the X-Forwarded-For Serve sets and Caddy keeps names a non-loopback
14
+ address (a probe of 127.0.0.1:8185, or of Caddy from this host, carries none)
15
+ no-identity enrold.caller_of(headers).present: Serve's Tailscale-User-Login and -Name header set (a tagged
16
+ device has none)
17
+ invalid-login the login validates as members.py stores one
18
+ not-pinned enrold.read_mode()[0] == 'request': the operator ran the RUNBOOK's Serve header check and pinned
19
+ `lanes enrol mode request`. Unpinned (auto), pinned `invite` or unreadable: refused — docsd never
20
+ latches anything from traffic and never reads LANES_ENROL_REQUEST_MODE
21
+ not-a-member login.casefold() in enrold.member_logins(): the login owns a valid members/<device>.json
22
+ domain-not-allowed enrold's domain rule over LANES_DOCS_DOMAINS (else LANES_ENROL_DOMAINS; default beyondwork.ai;
23
+ comma/space separated, `*` = any)
24
+ cross-site fetch_site_ok, on EVERY route (security review 2026-10-08, F2): the request's Sec-Fetch-Site
25
+ is `same-origin` or `none` (a page, script, stylesheet or image of this site, or a navigation the
26
+ person started: the address bar, a bookmark, a link opened from a terminal or another app) — or
27
+ it is a top-level navigation, a GET or HEAD with Sec-Fetch-Mode `navigate` and Sec-Fetch-Dest
28
+ `document`, whatever its Sec-Fetch-Site (a link tapped in ntfy's web app, a chat or a mail).
29
+ Everything else from another site is refused: a sandboxed artifact is an opaque origin, so its
30
+ fetch() and subresource loads are `cross-site` (mode cors/no-cors, dest empty/image/script/…),
31
+ and *.ts.net is a public suffix, so another tailnet host's page is `same-site`; neither may spend
32
+ the viewer's login here. POST /up keeps the stricter rule: `same-origin` or `none` only, a
33
+ navigation included (a form posted from another site). An absent Sec-Fetch-Site is refused
34
+ unless the request is such a navigation: every browser that reaches Serve sends the headers
35
+ (Chrome 76, Firefox 90, Safari 16.4). The artifact's sandbox (`sandbox allow-scripts`, nothing
36
+ else: no allow-popups, allow-top-navigation, allow-forms or allow-downloads) cannot open a
37
+ window, submit a form or navigate another browsing context; the one navigation its script can
38
+ start is of its own page, which replaces it (its script ends) unless the answer is a download the
39
+ sandbox then blocks — so what it can request through the navigation exemption is bounded by the
40
+ per-login limit below (VIEWS per VIEW_WINDOW; the HTML standard's rules, not measured in a
41
+ browser).
42
+ No Zellij cookie is consulted, ever. DEF-1645 (a process that reaches loopback can send the headers itself) stays the
43
+ documented open risk; the pin gate and the Serve-only check are the controls. Never put this behind Tailscale Funnel.
44
+ A refusal is one docs.log `denied` line per source (the tailnet address Serve forwarded, else the peer) and word; the
45
+ same source refused again within DENIED_REPEAT_S adds no line, and one `denied-repeat` line per DENIED_REPEAT_S says how
46
+ many it folded (its `bytes` field is that count); the login is logged at most MAX_DENIED_LOGIN characters (F4).
47
+
48
+ Limits (security review 2026-10-08, F2/F5/F8; per process): an authenticated login makes at most VIEWS requests other
49
+ than POST /up in VIEW_WINDOW s (more: 429 rate-limited + Retry-After, one docs.log line per window); at most
50
+ RENDER_SLOTS Markdown renders run at once (a view waits RENDER_WAIT_S for one, then 503 busy); a rendered Markdown
51
+ document is kept in memory by its sha256, least recently used out first, at most RENDER_CACHE_BYTES in all and
52
+ RENDER_CACHE_ENTRY each, so viewing it again renders nothing; UPLOAD_SLOTS big buffers at once, shared by uploads and
53
+ decryptions, and at most one upload per login; an upload's body must arrive within BODY_GRACE_S + its length /
54
+ BODY_MIN_RATE seconds, else 408 request-timeout and its slot is free again.
55
+
56
+ Who sees a document (visible). docs_store's own rule lets a login see what its store holds, what `for` names it in,
57
+ and every document whose `for` is empty; docsd narrows the last case. A document is visible to the caller when
58
+ * it is in the caller's own store (docs/<stem(caller)>/…: an upload of theirs), or
59
+ * its `for` is not empty and names the caller (casefolded), or
60
+ * its `for` is empty and a member record the caller owns grants the document's workspace (member_workspaces): a
61
+ record's `workspaces` (the board ids an invite recorded with `lanes invite --workspaces`) grants those boards, and
62
+ a record that lists none (empty or absent: a key added by hand, a request, an invite minted without
63
+ `--workspaces`) grants every workspace this host serves (the registry's). Nothing else in Lanes narrows by that
64
+ list — a view shows every board — so an empty list never meant "no board" (DEF-1852). The upload form's lanes
65
+ (upload_targets) and `POST /up` follow the same grant.
66
+ A document the caller cannot see is answered 404, exactly like one that does not exist.
67
+
68
+ Publish pick-up (plan W10 W4; Pickup, a daemon thread main() starts, on its own PICKUP_INTERVAL = 5 s cycle — never
69
+ statusd's). `lane publish <file>` and lane-mcp's `lanes_publish` write ~/.lz/state/publish/<lane>/<id>.json in a
70
+ workspace container (DM-LANES-PUBLISH-REQUEST, lane_core.publish). Each cycle, for every container containers.json
71
+ names whose init pid is alive, whose /proc/<pid>/root is not the host's own `/` (a recycled pid), and — when this
72
+ account can see the home volume containers.json records for it — whose /home/coder is that volume (same st_dev and
73
+ st_ino: a pid the kernel gave another container is skipped; F10), the pick-up first surveys the container's publish
74
+ directory and its lane directories (os.scandir, opened without following a link, `L.open_real_dir`): at most
75
+ MAX_ENTRIES entries in all are examined per container per cycle, nothing more is read, and a container that holds more
76
+ is refused for the cycle — nothing is taken from it, and the journal says so once per OVER_CAP_SAY_S (F1: listing
77
+ 100,000 planted lane directories took 5.7 s and 116 MiB a cycle; ~650,000 crossed MemoryMax). Then, lane by lane, at
78
+ most MAX_PICKUPS requests per container:
79
+ 1. claim: `<id>.json` is renamed to `.<id>.taking` in the same directory (nothing is read from a directory docsd
80
+ cannot write: a request it could not remove would be taken again every cycle). A directory planted at the claim's
81
+ name is removed when empty, else moved aside to a dot name; a request that still cannot be claimed is skipped and
82
+ the next one is taken (F9: one planted `.taking` directory stalled its lane's queue);
83
+ 2. read it as a regular file of at most MAX_REQUEST bytes (`L.read_regular_at`: never a link or a FIFO) and check
84
+ it: `v` 1, `path` absolute, `kind` md|html|svg|file, `claimed_sha256` 64 hex, `bytes` <= PUBLISH_LIMIT (the
85
+ docs limit, 32 MiB), `title`/`for`/`mime` strings or null; the lane must be one the registry places in THIS
86
+ container; PUBLISHES requests per container per PUBLISH_WINDOW s;
87
+ 3. resolve `path` inside the container (`L.container_realpath`: a link restarts at the container's root, never the
88
+ host's) to a file under /home/coder, open every directory without following a link (`L.open_real_dir`) and the
89
+ file O_NOFOLLOW|O_NONBLOCK, a regular file of at most PUBLISH_LIMIT bytes: a link swapped in meanwhile, a FIFO,
90
+ a device or a larger file is refused, so a container never makes the host read outside its home;
91
+ 4. stream it into docs_store.intake(login, ws, lane, name, …, limit=PUBLISH_LIMIT, source='publish', kind, title,
92
+ for_logins, by=<lane id>) through a reader that hashes as it goes and, at the end, refuses a sha256 other than
93
+ the claimed one inside intake (which then removes what it wrote: nothing is stored);
94
+ 5. write `<id>.receipt.json` back beside the request (`L.atomic_write_at`, 0644; the statusd write_container_status
95
+ pattern: replaced by rename, never written through a link; a directory planted at that name is removed or moved
96
+ aside first, F9; DM-LANES-PUBLISH-RECEIPT) — {v: 1, id, request, sha256, bytes, url, at, outcome: ok} or {…, id:
97
+ null, url: null, outcome: refused, reason} — remove the claim, keep the newest MAX_RECEIPTS receipts of the lane,
98
+ and append one docs.log `publish` line (outcome `ok` or the reason). The counts follow through intake. A busy
99
+ store leaves the claim's request for the next cycle.
100
+ Refusal reasons (docs.log): unreadable-request, bad-version, bad-request, bad-kind, bad-title, too-large, unknown-lane,
101
+ rate-limited, for-not-a-member, no-recipient, outside-home, not-a-file, sha256-mismatch, invalid (intake refused),
102
+ error (anything else after the claim: the journal has it). The receipt says the same word, except that
103
+ for-not-a-member and no-recipient both say `not-delivered` (F6: the two words told a container whether a login it
104
+ named owns a member record; a `for` naming a member still answers ok). Without a registry.json nothing is taken (a
105
+ request waits).
106
+
107
+ Who a published document is for (recipient): the request's `for` — a login that owns a member record (else refused
108
+ `for-not-a-member`), stored in that login's store with `for` = [it]; else the member whose profile's `last_opened.lane`
109
+ is this lane, the latest `at` winning (users/<login>.json), the same way; else every member whose record grants the
110
+ lane's workspace (member_workspaces' rule): `for` empty, stored in the store of the first of them by casefolded login
111
+ (none: refused `no-recipient`). Logins are spelled as their member record spells them. A successful pick-up sends one
112
+ `published` push through notify (notify.notify_published: notify.conf, the desk-presence rule, at most one per lane
113
+ per PUBLISHED_PUSH_S), best effort, after the cycle's receipts are written.
114
+
115
+ Wire contract v1 (DM-LANES-DOCS-WIRE; paths as docsd sees them, after Caddy strips /lanes-docs; every link and asset
116
+ reference a page carries is RELATIVE, so the pages work under that prefix):
117
+ GET / 200 page: the caller's visible documents, workspace -> lane -> newest first, with counts;
118
+ the upload form (the lanes of the caller's workspaces); an empty state
119
+ GET /?ws=<ws>&lane=<lane> the same, only that workspace and/or lane (the sidebar's and the shells' deep link for a
120
+ lane is exactly `/lanes-docs/?ws=<ws>&lane=<lane>`, W7). Either may be left out. A value
121
+ must be known to the caller (index_filter: a workspace a member record of theirs lists,
122
+ or one a visible document is in; a lane of the registry in such a workspace, or one a
123
+ visible document is in, and in the named workspace when both are given) — anything
124
+ else, a repeated or empty value included, is 404 not-found
125
+ GET /d/<id> 200 page by kind: md -> mdlite.render() in the page (503 busy + Retry-After when no
126
+ render slot frees within RENDER_WAIT_S; a document rendered before comes from the cache);
127
+ html|svg -> title, facts and a link to d/<id>/artifact (never embedded, no iframe);
128
+ upload|file -> facts and a download link
129
+ GET /d/<id>/raw 200 the stored bytes: Content-Disposition: attachment; filename="<name>", X-Lanes-Sha256
130
+ GET /d/<id>/artifact 200 for kind html|svg only (else 404): the bytes as a top-level document with
131
+ Content-Security-Policy: sandbox allow-scripts (nothing else) — an opaque origin
132
+ An ENCRYPTED document (docs_store's `age` switch, meta `enc`): raw and artifact decrypt
133
+ it on the fly when LANES_DOCS_AGE_IDENTITY names an identity file (at most MAX_BODY
134
+ bytes, one at a time, checked against the receipt), else 409 encrypted (a page: fetch
135
+ the copy in the workspace's uploads/); its view shows the facts, Markdown rendered only
136
+ when it can be decrypted
137
+ GET /receipt/<id>.json 200 the document's receipt.json bytes (DM-LANES-DOCS-RECEIPT), application/json
138
+ GET /docs.css, /docs.js 200 lanes/web/docs.css|docs.js of the tree this process runs (W6 writes them), else 404
139
+ GET /upload.css, /upload.js 200 lanes/web/upload.css|upload.js (W5: the index's drop area), else 404
140
+ POST /up multipart/form-data: `lane` (a lane id of the registry), optional `ws` (must be that
141
+ lane's workspace), one or more file parts (a part with a filename). The workspace must be
142
+ one the caller's member records grant. Each file goes through uploads.receive (W5): it
143
+ lands in the workspace's uploads/ folder in its container (`<folder>/uploads/<name>`,
144
+ name docs_store.clean_name's, `-1` on a collision, exactly as `lanes upload` lands one)
145
+ AND docs_store.intake(login, ws, lane, name, …, source='web', kind by extension (md,
146
+ html, svg, else file), for=[caller], by=caller) in one pass, both or neither, and is
147
+ answered with a receipt {id, name, sha256, bytes, url, at, landed} (`landed` the
148
+ in-container path an agent reads): 200 JSON {receipts} (an
149
+ Accept without text/html) or a page. Refused: 429 rate-limited + Retry-After (UPLOADS
150
+ per login in UPLOAD_WINDOW s, counted when the attempt starts), 503 busy + Retry-After
151
+ (UPLOAD_SLOTS uploads and decryptions at once, one per login, or the store lock), 415
152
+ not multipart/form-data, 411 no Content-Length (or a chunked body), 413 over MAX_BODY (32
153
+ MiB), 408 request-timeout (the body did not arrive within BODY_GRACE_S + length /
154
+ BODY_MIN_RATE s), 400 bad-request (a
155
+ malformed form) | no-file | no-lane | unknown-lane | unknown-workspace, 503
156
+ registry-unavailable (no registry.json to resolve the lane in), 503 uploads-unavailable
157
+ (the workspace's container folder cannot be written: not running, a link on the way),
158
+ 500 encrypt-failed (the `age` switch is on and age failed). The gate's `cross-site`
159
+ (403) applies to an upload with no navigation exemption: same-origin or none only
160
+ every route but POST /up 429 rate-limited + Retry-After past VIEWS requests per login in VIEW_WINDOW s
161
+ everything else 404 not-found (an id is matched against docs_store's id rule before any path is formed;
162
+ `..`, encoded dots and overlong ids never match); a known path with another method 405
163
+ Errors are JSON {"error": <word>}, or a short page when the Accept header names text/html. A request body left unread
164
+ by a refusal is read away up to its Content-Length (at most DRAIN_LIMIT) BEFORE the answer, and the answer carries
165
+ `Connection: close`; past DRAIN_LIMIT the answer is sent, the write side shut and the rest read away for LINGER_S
166
+ seconds, so the client still reads the refusal instead of a reset (DEF-1854).
167
+
168
+ Headers. Every response: Cache-Control no-store (static files no-cache), X-Content-Type-Options nosniff,
169
+ X-Frame-Options DENY, Referrer-Policy no-referrer and a Content-Security-Policy: a page PAGE_POLICY, an artifact
170
+ exactly ARTIFACT_POLICY, everything else DATA_POLICY. The site-wide Caddy header block skips /lanes-docs/* (`header
171
+ @notdocs`), so these are the only ones a document gets.
172
+
173
+ docs.log (DM-LANES-DOCS-LOG, through docs_store.audit; never content, a title or a name): `denied` (a gate refusal,
174
+ the outcome is its word; coalesced per source, with `denied-repeat` lines, see the gate above), `view` `raw` `artifact`
175
+ `receipt` (ok | not-found | encrypted | busy | rate-limited), `upload` (a refusal before any file reaches intake;
176
+ intake writes its own `intake` line per file) and `publish` (one per request the pick-up takes: the store login, the
177
+ document id, the lane, the host's bytes and sha256 when known, `ok` or the refusal reason).
178
+ """
179
+ import argparse
180
+ import collections
181
+ import email.message
182
+ import email.parser
183
+ import email.policy
184
+ import errno
185
+ import hashlib
186
+ import html
187
+ import ipaddress
188
+ import json
189
+ import os
190
+ import re
191
+ import socket
192
+ import socketserver
193
+ import stat
194
+ import sys
195
+ import threading
196
+ import time
197
+ import traceback
198
+ import urllib.parse
199
+ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
200
+ from pathlib import Path
201
+
202
+ sys.path.insert(0, str(Path(__file__).resolve().parent))
203
+ import lanes_lib as L # noqa: E402
204
+ import docs_store as D # noqa: E402
205
+ import enrold as E # noqa: E402
206
+ import keys as K # noqa: E402
207
+ import mdlite # noqa: E402
208
+ import notify as N # noqa: E402
209
+ import uploads as U # noqa: E402
210
+ import users # noqa: E402
211
+
212
+ WIRE_VERSION = 1
213
+ DEFAULT_PORT = 8185 # web/Caddyfile `handle_path /lanes-docs/*` forwards here
214
+ DEFAULT_BIND = '127.0.0.1'
215
+ MAX_BODY = 32 * 1024 * 1024 # a POST /up body; also each file's intake limit (uploads.UPLOAD_LIMIT)
216
+ DRAIN_LIMIT = 2 * MAX_BODY # an unread body is read away up to this BEFORE a refusal is sent (DEF-1854)
217
+ LINGER_S = 5 # past DRAIN_LIMIT: seconds the rest is read away after the answer
218
+ MAX_PARTS = 64 # parts of one form (fields and files)
219
+ MAX_PART_HEADER = 8 * 1024 # one part's header block
220
+ MAX_FIELD = 1024 # one non-file field's value
221
+ MAX_CONTENT_TYPE = 512
222
+ MAX_STATIC = 512 * 1024 # docs.css / docs.js
223
+ UPLOADS = 30 # POST /up attempts per login in UPLOAD_WINDOW seconds
224
+ UPLOAD_WINDOW = 600
225
+ UPLOAD_SLOTS = 2 # big buffers at once (<= MAX_BODY each): uploads parsed and documents decrypted
226
+ BUSY_RETRY = 5 # Retry-After of a 503 busy
227
+ DECRYPT_WAIT_S = BUSY_RETRY # a decryption waits this long for its turn and a slot, then 503 busy
228
+ MAX_LOGINS = 4096 # logins a limiter remembers
229
+ BODY_GRACE_S = 30 # an upload body must arrive within BODY_GRACE_S + its length / BODY_MIN_RATE s
230
+ BODY_MIN_RATE = 64 * 1024 # (bytes per second: 32 MiB in 542 s), else 408 and the slot is free (F5)
231
+ VIEWS = 300 # requests other than POST /up per login in VIEW_WINDOW seconds (F2)
232
+ VIEW_WINDOW = 60
233
+ RENDER_SLOTS = 2 # Markdown renders at once: a hostile 1 MiB document renders for seconds (F2)
234
+ RENDER_WAIT_S = 1.0 # a view waits this long for a render slot, then 503 busy
235
+ RENDER_CACHE_BYTES = 8 * 1024 * 1024 # rendered Markdown kept in memory by the document's sha256, least recent out
236
+ RENDER_CACHE_ENTRY = 2 * 1024 * 1024 # a larger render is not kept
237
+ DENIED_REPEAT_S = 60 # one docs.log line per refused source and word in this many seconds (F4)
238
+ MAX_DENIED_SOURCES = 1024 # sources the refusal coalescing remembers
239
+ MAX_DENIED_LOGIN = 64 # characters of an unauthenticated login a `denied` line keeps
240
+ CHUNK = 64 * 1024
241
+ PAGE_POLICY = ("default-src 'none'; style-src 'self' 'unsafe-inline'; script-src 'self'; img-src 'self' data:; "
242
+ "connect-src 'self'; form-action 'self'; base-uri 'none'") # connect-src: upload.js's POST /up
243
+ ARTIFACT_POLICY = 'sandbox allow-scripts'
244
+ DATA_POLICY = "default-src 'none'; sandbox"
245
+ SHA_HEADER = 'X-Lanes-Sha256'
246
+ ARTIFACT_TYPES = {'html': 'text/html; charset=utf-8', 'svg': 'image/svg+xml'}
247
+ STATIC = {'docs.css': 'text/css; charset=utf-8', 'docs.js': 'text/javascript; charset=utf-8',
248
+ 'upload.css': 'text/css; charset=utf-8', 'upload.js': 'text/javascript; charset=utf-8'}
249
+ FETCH_SITES = ('same-origin', 'none') # Sec-Fetch-Site values the gate serves (every route, F2) …
250
+ NAVIGATION_METHODS = ('GET', 'HEAD') # … and a top-level navigation by these methods, from any site
251
+ _DECRYPTS = threading.BoundedSemaphore(1) # encrypted documents decrypted at once (each held in memory)
252
+ _ID = r'([a-z0-9]{10,20})' # docs_store.ID_RE; check_id runs again before any path is formed
253
+ ROUTES = (('index', re.compile(r'/', re.ASCII)),
254
+ ('view', re.compile(rf'/d/{_ID}', re.ASCII)),
255
+ ('raw', re.compile(rf'/d/{_ID}/raw', re.ASCII)),
256
+ ('artifact', re.compile(rf'/d/{_ID}/artifact', re.ASCII)),
257
+ ('receipt', re.compile(rf'/receipt/{_ID}\.json', re.ASCII)),
258
+ ('up', re.compile(r'/up', re.ASCII)),
259
+ ('static', re.compile(r'/(docs\.css|docs\.js|upload\.css|upload\.js)', re.ASCII)))
260
+ METHODS = {'up': ('POST',)} # every other route: GET
261
+ _MIME = re.compile(r'[a-z0-9][a-z0-9.+-]{0,63}/[a-z0-9][a-z0-9.+-]{0,63}', re.ASCII)
262
+ _FILENAME_UNSAFE = re.compile(r'[^A-Za-z0-9._-]')
263
+ _BOUNDARY = re.compile(r"[0-9A-Za-z'()+_,./:=? -]{0,69}[0-9A-Za-z'()+_,./:=?-]", re.ASCII) # RFC 2046 bchars
264
+ _CTE_OK = ('7bit', '8bit', 'binary')
265
+ _CONTROL = re.compile(r'[\x00-\x1f\x7f]')
266
+
267
+ SENTENCES = {
268
+ 'not-through-serve': 'This page answers only requests that come through Tailscale Serve, at the server\'s '
269
+ 'tailnet HTTPS address.',
270
+ 'no-identity': 'No tailnet identity reached this page: open it from a device signed in to the tailnet as yourself '
271
+ '(a tagged device has no login).',
272
+ 'invalid-login': 'The tailnet login this request carries is not one Lanes accepts.',
273
+ 'not-pinned': 'The docs service is off until the operator has run the Serve header check and pinned it '
274
+ '(`lanes enrol mode request` on the server).',
275
+ 'not-a-member': 'Your tailnet login owns no member record on this server.',
276
+ 'domain-not-allowed': 'Your tailnet login is not in a domain this server accepts.',
277
+ 'not-found': 'No such document here, or it is not one you can open.',
278
+ 'method-not-allowed': 'That method is not used here.',
279
+ 'cross-site': 'Refused: this request came from another site and was not a link you followed (Sec-Fetch-Site), '
280
+ 'or your browser does not say where it came from. Open the Docs page from the shell or the sidebar '
281
+ 'link, or type its address.',
282
+ 'rate-limited': 'Too many requests in a short time; try again in a few minutes.',
283
+ 'busy': 'The docs service is busy; try again in a moment.',
284
+ 'request-timeout': 'The upload arrived too slowly and was stopped; nothing was stored. Try again.',
285
+ 'unsupported-media-type': 'An upload is a multipart/form-data form.',
286
+ 'length-required': 'An upload needs a Content-Length.',
287
+ 'too-large': 'The upload is larger than 32 MiB.',
288
+ 'bad-request': 'That upload form could not be read.',
289
+ 'no-file': 'The form carried no file.',
290
+ 'no-lane': 'Choose the lane the file is for.',
291
+ 'unknown-lane': 'That lane is not one of this server\'s lanes.',
292
+ 'unknown-workspace': 'That workspace is not one your member record lists.',
293
+ 'registry-unavailable': 'The server has no lane registry yet (`lanes sync`).',
294
+ 'uploads-unavailable': 'The workspace\'s uploads/ folder cannot be written now (its container is not running, or '
295
+ 'the folder is reached through a link); nothing was stored.',
296
+ 'encrypt-failed': 'The docs store encrypts at rest and age failed; the journal says why. Nothing was stored.',
297
+ 'encrypted': 'This document is encrypted at rest and the docs service holds no key to open it: fetch the copy in '
298
+ 'the workspace\'s uploads/ folder (or the lane\'s own file) instead.',
299
+ 'server-error': 'Something went wrong on the server; the journal says what.',
300
+ }
301
+
302
+
303
+ # ------------------------------------------------------------------------------------------------------- basics
304
+
305
+ def version():
306
+ return E.version()
307
+
308
+
309
+ def web_dir():
310
+ """lanes/web of the tree this process runs (L.REPO: the unit names the `current` link, so the installed release),
311
+ where docs.css and docs.js live (W6)."""
312
+ return L.REPO / 'web'
313
+
314
+
315
+ def esc(value):
316
+ return html.escape('' if value is None else str(value), quote=True)
317
+
318
+
319
+ def human_bytes(size):
320
+ if size < 1024:
321
+ return f'{size} B'
322
+ if size < 1024 * 1024:
323
+ return f'{size / 1024:.1f} KiB'
324
+ return f'{size / (1024 * 1024):.1f} MiB'
325
+
326
+
327
+ def wants_html(headers):
328
+ return 'text/html' in (headers.get('Accept') or '').lower()
329
+
330
+
331
+ def download_name(name):
332
+ """The filename a Content-Disposition carries: docs_store names are already [A-Za-z0-9._-]; anything else is `_`."""
333
+ cleaned = _FILENAME_UNSAFE.sub('_', name or '')[:D.MAX_NAME + D.NAME_SLACK]
334
+ return cleaned or 'file'
335
+
336
+
337
+ def content_type_of(meta):
338
+ mime = meta.get('mime')
339
+ return mime if isinstance(mime, str) and _MIME.fullmatch(mime) else 'application/octet-stream'
340
+
341
+
342
+ # ------------------------------------------------------------------------------------------------------- people
343
+
344
+ def served_workspaces(reg):
345
+ """The workspace ids the registry `reg` serves -> a set (empty without a registry)."""
346
+ return {workspace['id'] for workspace in (reg or {}).get('workspaces') or []
347
+ if isinstance(workspace, dict) and isinstance(workspace.get('id'), str)}
348
+
349
+
350
+ def _listed(member):
351
+ return {ws for ws in member.get('workspaces') or () if isinstance(ws, str)}
352
+
353
+
354
+ def member_workspaces(login, reg=None):
355
+ """The workspaces the member records `login` owns grant it (owner login compared casefolded) -> a set: each
356
+ record's `workspaces`, and every workspace the host serves (served_workspaces of `reg`, else registry.json) for a
357
+ record that lists none — empty or absent (DEF-1852; the module docstring has the rule). The records
358
+ keys.load_members accepts, as enrold.member_logins reads them."""
359
+ members, _problems = K.load_members()
360
+ folded = login.casefold()
361
+ found, every = set(), False
362
+ for member in members:
363
+ owner = member.get('owner')
364
+ if isinstance(owner, dict) and owner.get('login', '').casefold() == folded:
365
+ listed = _listed(member)
366
+ found |= listed
367
+ every = every or not listed
368
+ if every:
369
+ found |= served_workspaces(registry() if reg is None else reg)
370
+ return found
371
+
372
+
373
+ def member_grants(reg):
374
+ """{login casefolded: (the login as its member record spells it, the workspaces its records grant)} for every
375
+ member at once (member_workspaces' rule, `reg` the registry): the pick-up's view of who may receive a document."""
376
+ members, _problems = K.load_members()
377
+ served = served_workspaces(reg)
378
+ out = {}
379
+ for member in members:
380
+ owner = member.get('owner')
381
+ login = owner.get('login') if isinstance(owner, dict) else None
382
+ if not isinstance(login, str):
383
+ continue
384
+ spelling, granted = out.get(login.casefold(), (login, set()))
385
+ out[login.casefold()] = (spelling, granted | (_listed(member) or served))
386
+ return out
387
+
388
+
389
+ def in_own_store(meta, login):
390
+ """Whether the document `meta` describes sits in `login`'s own store: docs/<stem(login)>/<ws>/<lane>/<id> is a
391
+ real directory (never through a link)."""
392
+ try:
393
+ stem = D.login_stem(login)
394
+ except L.LanesError:
395
+ return False
396
+ try:
397
+ info = os.lstat(D.root() / stem / meta['ws'] / meta['lane'] / meta['id'])
398
+ except (OSError, KeyError, TypeError):
399
+ return False
400
+ return stat.S_ISDIR(info.st_mode)
401
+
402
+
403
+ def visible(meta, owner, login, workspaces):
404
+ """docsd's visibility rule (see the module docstring): the caller's own store; else `for` names the caller; else
405
+ (`for` empty) a member record of the caller lists the document's workspace."""
406
+ if owner:
407
+ return True
408
+ fors = meta.get('for') or []
409
+ if fors:
410
+ folded = login.casefold()
411
+ return any(isinstance(entry, str) and entry.casefold() == folded for entry in fors)
412
+ return meta.get('ws') in workspaces
413
+
414
+
415
+ def visible_docs(login, workspaces):
416
+ """The metas `login` may see, newest first."""
417
+ return [meta for meta in D.list_for(login) if visible(meta, in_own_store(meta, login), login, workspaces)]
418
+
419
+
420
+ def find_doc(doc_id, login, workspaces):
421
+ """(meta, (stem, ws, lane, id, name)) of a document the caller may see, else None."""
422
+ D.check_id(doc_id)
423
+ found = D.load(doc_id, login)
424
+ if found is None:
425
+ return None
426
+ meta, path = found
427
+ parts = path.relative_to(D.root()).parts
428
+ if len(parts) != 5:
429
+ return None
430
+ try:
431
+ owner = parts[0] == D.login_stem(login)
432
+ except L.LanesError:
433
+ owner = False
434
+ return (meta, parts) if visible(meta, owner, login, workspaces) else None
435
+
436
+
437
+ def open_doc(meta, parts):
438
+ """A read-only fd of the document file: every directory component a real directory, the file never a link, never a
439
+ FIFO, a regular file of the recorded size. Raises OSError otherwise. The caller closes it."""
440
+ directory = L.open_real_dir(D.root(), parts[:4])
441
+ try:
442
+ fd = os.open(parts[4], os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK | getattr(os, 'O_CLOEXEC', 0),
443
+ dir_fd=directory)
444
+ finally:
445
+ os.close(directory)
446
+ try:
447
+ info = os.fstat(fd)
448
+ if not stat.S_ISREG(info.st_mode) or info.st_size != D.stored_file(meta)[1]:
449
+ raise OSError('not the recorded document')
450
+ except BaseException:
451
+ os.close(fd)
452
+ raise
453
+ return fd
454
+
455
+
456
+ def read_receipt(parts):
457
+ """receipt.json's bytes beside the document, or None (absent, a link, not a regular file, too large)."""
458
+ try:
459
+ directory = L.open_real_dir(D.root(), parts[:4])
460
+ except (OSError, L.LanesError):
461
+ return None
462
+ try:
463
+ return L.read_regular_at(directory, D.RECEIPT, D.MAX_META)
464
+ finally:
465
+ os.close(directory)
466
+
467
+
468
+ def read_all(fd, limit):
469
+ chunks, size = [], 0
470
+ while size <= limit:
471
+ chunk = os.read(fd, min(CHUNK, limit + 1 - size))
472
+ if not chunk:
473
+ break
474
+ chunks.append(chunk)
475
+ size += len(chunk)
476
+ return b''.join(chunks)
477
+
478
+
479
+ def registry():
480
+ """registry.json, or None when there is none to read."""
481
+ try:
482
+ return L.load_registry()
483
+ except (L.LanesError, OSError, ValueError):
484
+ return None
485
+
486
+
487
+ def upload_targets(workspaces, reg):
488
+ """[(ws id, ws title, [(lane id, lane name)])] — the registry's lanes of the caller's workspaces, catalog order."""
489
+ targets = []
490
+ for workspace in (reg or {}).get('workspaces') or []:
491
+ if not isinstance(workspace, dict) or workspace.get('id') not in workspaces:
492
+ continue
493
+ lanes = [(lane['id'], lane.get('name') or lane['id']) for lane in workspace.get('lanes') or []
494
+ if isinstance(lane, dict) and isinstance(lane.get('id'), str)]
495
+ if lanes:
496
+ targets.append((workspace['id'], workspace.get('title') or workspace['id'], lanes))
497
+ return targets
498
+
499
+
500
+ def index_filter(query, workspaces, docs, targets):
501
+ """(ws, lane) a `GET /?ws=…&lane=…` query names (None for one left out). Known to the caller: a workspace its member
502
+ records list or a visible document is in; a lane of the registry in such a workspace (`targets`) or one a visible
503
+ document is in — and of the named workspace when both are given. Raises _Reply(404) for anything else (unknown,
504
+ repeated, empty, or more fields than a filter has); other keys are ignored."""
505
+ try:
506
+ params = urllib.parse.parse_qs(query or '', keep_blank_values=True, max_num_fields=8)
507
+ except ValueError:
508
+ raise _Reply(404, 'not-found') from None
509
+ picked = {}
510
+ for key in ('ws', 'lane'):
511
+ values = params.get(key)
512
+ if values is None:
513
+ picked[key] = None
514
+ elif len(values) != 1 or not values[0]:
515
+ raise _Reply(404, 'not-found')
516
+ else:
517
+ picked[key] = values[0]
518
+ known = {ws: set() for ws in workspaces}
519
+ for ws, _title, lanes in targets:
520
+ known.setdefault(ws, set()).update(lane for lane, _name in lanes)
521
+ for meta in docs:
522
+ known.setdefault(meta['ws'], set()).add(meta['lane'])
523
+ ws, lane = picked['ws'], picked['lane']
524
+ if ws is not None and ws not in known:
525
+ raise _Reply(404, 'not-found')
526
+ if lane is not None and lane not in (known[ws] if ws is not None else set().union(*known.values())):
527
+ raise _Reply(404, 'not-found')
528
+ return ws, lane
529
+
530
+
531
+ # ---------------------------------------------------------------------------------------------------- the form
532
+
533
+ class FormError(Exception):
534
+ def __init__(self, error):
535
+ super().__init__(error)
536
+ self.error = error
537
+
538
+
539
+ def form_boundary(content_type):
540
+ """The boundary of a `multipart/form-data` Content-Type (read with email.message), or None."""
541
+ if not content_type or len(content_type) > MAX_CONTENT_TYPE or _CONTROL.search(content_type):
542
+ return None
543
+ message = email.message.Message()
544
+ message['Content-Type'] = content_type
545
+ if message.get_content_type() != 'multipart/form-data':
546
+ return None
547
+ boundary = message.get_param('boundary')
548
+ if not isinstance(boundary, str) or not _BOUNDARY.fullmatch(boundary):
549
+ return None
550
+ return boundary
551
+
552
+
553
+ def parse_form(body, boundary):
554
+ """A multipart/form-data body -> (fields {name: str}, files [(filename, start, end)]) with the files as offsets into
555
+ `body` (no copy). The delimiters are found by a linear bytes scan (RFC 2046: CRLF "--" boundary, the last one
556
+ followed by "--"); each part's header block is read by email.parser (Content-Disposition form-data, its name and
557
+ filename — filename* included — and a Content-Transfer-Encoding of 7bit/8bit/binary or none). Parsing the whole
558
+ body with email.parser instead held about 13 times a 32 MiB body in memory (measured, see RUNBOOK). A part with an
559
+ empty filename and no bytes (a file input left empty) is skipped. Raises FormError('bad-request')."""
560
+ delimiter = b'--' + boundary.encode('ascii')
561
+ if body.startswith(delimiter):
562
+ at = len(delimiter)
563
+ else: # a preamble is allowed; browsers send none
564
+ found = body.find(b'\r\n' + delimiter)
565
+ if found < 0:
566
+ raise FormError('bad-request')
567
+ at = found + 2 + len(delimiter)
568
+ fields, files, parts = {}, [], 0
569
+ parser = email.parser.BytesHeaderParser(policy=email.policy.HTTP)
570
+ while True:
571
+ if body[at:at + 2] == b'--': # the close delimiter: the epilogue is ignored
572
+ return fields, files
573
+ line_end = body.find(b'\r\n', at)
574
+ if line_end < 0 or body[at:line_end].strip(b' \t'): # transport padding only, then CRLF
575
+ raise FormError('bad-request')
576
+ start = line_end + 2
577
+ end = body.find(b'\r\n' + delimiter, start)
578
+ if end < 0:
579
+ raise FormError('bad-request')
580
+ parts += 1
581
+ if parts > MAX_PARTS:
582
+ raise FormError('bad-request')
583
+ if body[start:start + 2] == b'\r\n':
584
+ header, content = b'', start + 2
585
+ else:
586
+ header_end = body.find(b'\r\n\r\n', start, end)
587
+ if header_end < 0 or header_end - start > MAX_PART_HEADER:
588
+ raise FormError('bad-request')
589
+ header, content = body[start:header_end + 2], header_end + 4
590
+ part = parser.parsebytes(header)
591
+ if part.get_content_disposition() != 'form-data':
592
+ raise FormError('bad-request')
593
+ name = part.get_param('name', header='content-disposition')
594
+ encoding = (part.get('Content-Transfer-Encoding') or '7bit').strip().lower()
595
+ if not isinstance(name, str) or encoding not in _CTE_OK:
596
+ raise FormError('bad-request')
597
+ filename = part.get_filename()
598
+ if filename is not None or part.get_param('filename', header='content-disposition') is not None:
599
+ if filename or end > content:
600
+ files.append((filename or '', content, end))
601
+ else:
602
+ if name in fields or end - content > MAX_FIELD:
603
+ raise FormError('bad-request')
604
+ try:
605
+ fields[name] = body[content:end].decode('utf-8')
606
+ except UnicodeDecodeError:
607
+ raise FormError('bad-request') from None
608
+ at = end + 2 + len(delimiter)
609
+
610
+
611
+ class _Slice:
612
+ """A reader over body[start:end] (docs_store.intake reads it in chunks; nothing is copied whole)."""
613
+
614
+ def __init__(self, data, start, end):
615
+ self._view, self._at = memoryview(data)[start:end], 0
616
+
617
+ def read(self, size=-1):
618
+ if size is None or size < 0:
619
+ size = len(self._view) - self._at
620
+ chunk = self._view[self._at:self._at + size]
621
+ self._at += len(chunk)
622
+ return bytes(chunk)
623
+
624
+
625
+ def upload_target(fields, workspaces, reg):
626
+ """(ws, lane) for a form's `lane` (and optional `ws`), validated against the registry and the caller's workspaces.
627
+ Raises FormError: no-lane, unknown-lane, unknown-workspace, registry-unavailable."""
628
+ lane = fields.get('lane')
629
+ if not lane:
630
+ raise FormError('no-lane')
631
+ try:
632
+ users.check_lane(lane)
633
+ except L.LanesError:
634
+ raise FormError('unknown-lane') from None
635
+ if reg is None:
636
+ raise FormError('registry-unavailable')
637
+ home = next((workspace.get('id') for workspace in reg.get('workspaces') or [] if isinstance(workspace, dict)
638
+ for entry in workspace.get('lanes') or [] if isinstance(entry, dict) and entry.get('id') == lane),
639
+ None)
640
+ if home is None:
641
+ raise FormError('unknown-lane')
642
+ named = fields.get('ws')
643
+ if (named is not None and named != home) or home not in workspaces:
644
+ raise FormError('unknown-workspace')
645
+ return home, lane
646
+
647
+
648
+ # ------------------------------------------------------------------------------------------------------- pages
649
+
650
+ def page(title, content, up='', assets=()):
651
+ """The page shell. `up` is the relative way back to the docs root ('' at /, '../' at /d/<id>): every reference is
652
+ relative, because Caddy serves this under /lanes-docs/ and strips the prefix. `assets` names extra stylesheet +
653
+ script pairs of web/ (the index's drop area: 'upload')."""
654
+ home = up or './'
655
+ extra = ''.join(f'<link rel="stylesheet" href="{up}{name}.css"><script src="{up}{name}.js" defer></script>'
656
+ for name in assets)
657
+ return ('<!doctype html><html lang="en"><head><meta charset="utf-8">'
658
+ '<meta name="viewport" content="width=device-width, initial-scale=1">'
659
+ '<meta name="referrer" content="no-referrer">'
660
+ f'<title>{esc(title)}</title>'
661
+ f'<link rel="stylesheet" href="{up}docs.css">'
662
+ f'<script src="{up}docs.js" defer></script>{extra}</head>'
663
+ f'<body class="lanes-docs"><header class="docs-top"><a class="docs-home" href="{home}">Docs</a></header>'
664
+ f'<main>{content}</main></body></html>').encode('utf-8')
665
+
666
+
667
+ def message_page(status, word):
668
+ sentence = SENTENCES.get(word, word)
669
+ return page('Docs', f'<h1>Docs</h1><p class="docs-note" data-error="{esc(word)}">{esc(sentence)}</p>'
670
+ f'<p class="docs-muted">{status} {esc(word)}</p>')
671
+
672
+
673
+ def _doc_row(meta):
674
+ label = meta.get('title') or meta['name']
675
+ return (f'<li class="docs-doc" data-kind="{esc(meta["kind"])}"><a href="d/{esc(meta["id"])}">{esc(label)}</a> '
676
+ f'<span class="docs-meta">{esc(meta["kind"])} · {esc(human_bytes(meta["bytes"]))} · '
677
+ f'<time>{esc(meta["at"])}</time></span></li>')
678
+
679
+
680
+ def index_page(login, docs, reg, targets, ws=None, lane=None):
681
+ """The index: `docs` (visible, newest first) narrowed to the filter `ws`/`lane` (index_filter checked them)."""
682
+ names = {}
683
+ titles = {}
684
+ for workspace in (reg or {}).get('workspaces') or []:
685
+ if isinstance(workspace, dict) and isinstance(workspace.get('id'), str):
686
+ titles[workspace['id']] = workspace.get('title') or workspace['id']
687
+ for entry in workspace.get('lanes') or []:
688
+ if isinstance(entry, dict) and isinstance(entry.get('id'), str):
689
+ names[entry['id']] = entry.get('name') or entry['id']
690
+ docs = [meta for meta in docs if (ws is None or meta['ws'] == ws) and (lane is None or meta['lane'] == lane)]
691
+ grouped = {}
692
+ for meta in docs: # newest first, so groups come newest first too
693
+ grouped.setdefault(meta['ws'], {}).setdefault(meta['lane'], []).append(meta)
694
+ parts = [f'<h1>Docs</h1><p class="docs-who">Signed in as {esc(login)} · {len(docs)} '
695
+ f'document{"" if len(docs) == 1 else "s"}</p>']
696
+ if ws is not None or lane is not None:
697
+ shown = ' › '.join(esc(label) for label in (titles.get(ws, ws) if ws else None,
698
+ names.get(lane, lane) if lane else None) if label)
699
+ parts.append(f'<p class="docs-filter">Showing {shown} · <a href="./">All docs</a></p>')
700
+ if targets:
701
+ options = ''.join(f'<optgroup label="{esc(title)}">' + ''.join(
702
+ f'<option value="{esc(lane_id)}"{" selected" if lane_id == lane else ""}>{esc(name)}</option>'
703
+ for lane_id, name in lanes) + '</optgroup>' for _ws, title, lanes in targets)
704
+ # The drop area (W5): a plain form that works without a script — the label wraps a multi-file input, so a tap
705
+ # opens the picker and a drop onto the input takes the files —, and better with upload.js (drag and drop
706
+ # onto the whole area, a size check, progress, one receipt per file in .docs-sent).
707
+ parts.append(f'<form class="docs-upload" method="post" action="up" enctype="multipart/form-data" '
708
+ f'data-max="{MAX_BODY}">'
709
+ f'<label>Lane <select name="lane" required>{options}</select></label>'
710
+ '<label class="docs-drop">Drop files here, or tap to choose'
711
+ '<input type="file" name="file" multiple required></label>'
712
+ '<button type="submit">Upload</button>'
713
+ f'<p class="docs-muted">Up to {human_bytes(MAX_BODY)} at once. Each file lands in the '
714
+ 'workspace\'s uploads/ folder, for the lane\'s agent, with a sha256 receipt here.</p>'
715
+ '<ol class="docs-sent" aria-live="polite"></ol></form>')
716
+ else:
717
+ parts.append('<p class="docs-note">Uploads go to a lane of a workspace your member record lists, and yours '
718
+ 'lists none here.</p>')
719
+ if not docs:
720
+ parts.append('<p class="docs-empty">No documents yet. An agent publishes one from a lane; an upload lands '
721
+ 'here with its receipt.</p>')
722
+ for ws_id, lanes in grouped.items():
723
+ total = sum(len(metas) for metas in lanes.values())
724
+ parts.append(f'<section class="docs-ws" data-ws="{esc(ws_id)}"><h2>{esc(titles.get(ws_id, ws_id))} '
725
+ f'<span class="docs-count">{total}</span></h2>')
726
+ for lane_id, metas in lanes.items():
727
+ parts.append(f'<h3 data-lane="{esc(lane_id)}">{esc(names.get(lane_id, lane_id))} '
728
+ f'<span class="docs-count">{len(metas)}</span></h3><ul class="docs-list">'
729
+ + ''.join(_doc_row(meta) for meta in metas) + '</ul>')
730
+ parts.append('</section>')
731
+ return page('Docs', ''.join(parts), assets=('upload',) if targets else ())
732
+
733
+
734
+ def _facts(meta):
735
+ rows = (('kind', meta['kind']), ('name', meta['name']), ('size', human_bytes(meta['bytes'])),
736
+ ('workspace', meta['ws']), ('lane', meta['lane']), ('at', meta['at']), ('by', meta.get('by') or '-'),
737
+ ('source', meta['source']), ('sha256', meta['sha256']))
738
+ return '<dl class="docs-facts">' + ''.join(f'<dt>{esc(key)}</dt><dd>{esc(value)}</dd>' for key, value in rows) \
739
+ + '</dl>'
740
+
741
+
742
+ def view_page(meta, body=None, locked=False):
743
+ """A document's page. `locked`: an encrypted document this service cannot open (no identity): its facts only."""
744
+ doc_id = esc(meta['id'])
745
+ title = meta.get('title') or meta['name']
746
+ links = (f'<p class="docs-links"><a href="{doc_id}/raw" download>Download</a> · '
747
+ f'<a href="../receipt/{doc_id}.json">Receipt</a></p>')
748
+ if locked:
749
+ content = f'<p class="docs-note" data-error="encrypted">{esc(SENTENCES["encrypted"])}</p>'
750
+ elif meta['kind'] == 'md':
751
+ if body is None:
752
+ content = (f'<p class="docs-note">This Markdown is larger than {human_bytes(mdlite.MAX_BYTES)} and is not '
753
+ 'rendered here; download it.</p>')
754
+ else:
755
+ content = f'<article class="docs-md">{body}</article>'
756
+ elif meta['kind'] in ARTIFACT_TYPES:
757
+ content = (f'<p class="docs-artifact"><a href="{doc_id}/artifact" rel="noopener noreferrer">Open the '
758
+ f'{esc(meta["kind"].upper())} artifact</a></p><p class="docs-muted">It opens on its own page in a '
759
+ 'sandbox: its scripts run, but cannot reach this site or your lanes.</p>')
760
+ else:
761
+ content = ''
762
+ return page(title, f'<h1>{esc(title)}</h1>{links}{content}{_facts(meta)}', up='../')
763
+
764
+
765
+ def receipts_page(receipts, error=None):
766
+ rows = ''.join(f'<li><a href="d/{esc(r["id"])}">{esc(r["name"])}</a> <span class="docs-meta">'
767
+ f'{esc(human_bytes(r["bytes"]))} · sha256 <code>{esc(r["sha256"])}</code> · in the lane as '
768
+ f'<code>{esc(r["landed"])}</code></span></li>' for r in receipts)
769
+ note = f'<p class="docs-note" data-error="{esc(error)}">{esc(SENTENCES.get(error, error))}</p>' if error else ''
770
+ return page('Uploaded', f'<h1>Uploaded</h1>{note}<ul class="docs-receipts">{rows}</ul>'
771
+ '<p><a href="./">Back to the docs</a></p>')
772
+
773
+
774
+ # ----------------------------------------------------------------------------------------------------- service
775
+
776
+ def fetch_site_ok(headers, method):
777
+ """The gate's `cross-site` rule (the module docstring): `Sec-Fetch-Site` same-origin or none, or a GET/HEAD
778
+ top-level navigation (`Sec-Fetch-Mode: navigate`, `Sec-Fetch-Dest: document`) from any site. A POST — an upload —
779
+ is never a navigation here."""
780
+ if headers.get('Sec-Fetch-Site') in FETCH_SITES:
781
+ return True
782
+ return (method in NAVIGATION_METHODS and headers.get('Sec-Fetch-Mode') == 'navigate'
783
+ and headers.get('Sec-Fetch-Dest') == 'document')
784
+
785
+
786
+ class RenderCache:
787
+ """Rendered Markdown by the document's sha256 (the same bytes render the same, mdlite is deterministic), least
788
+ recently used out first: at most RENDER_CACHE_BYTES in all, nothing larger than RENDER_CACHE_ENTRY kept. A value
789
+ of None is a document too large to render once its bytes are decoded."""
790
+
791
+ def __init__(self):
792
+ self._lock = threading.Lock()
793
+ self._entries = collections.OrderedDict()
794
+ self.size = 0
795
+
796
+ def get(self, key):
797
+ """(True, the render) when `key` is kept (it becomes the most recent), else (False, None)."""
798
+ with self._lock:
799
+ if key not in self._entries:
800
+ return False, None
801
+ self._entries.move_to_end(key)
802
+ return True, self._entries[key][0]
803
+
804
+ def put(self, key, value):
805
+ size = sys.getsizeof(value)
806
+ if size > RENDER_CACHE_ENTRY:
807
+ return
808
+ with self._lock:
809
+ if key in self._entries:
810
+ return
811
+ self._entries[key] = (value, size)
812
+ self.size += size
813
+ while self.size > RENDER_CACHE_BYTES and self._entries:
814
+ _key, (_value, gone) = self._entries.popitem(last=False)
815
+ self.size -= gone
816
+
817
+ def __len__(self):
818
+ return len(self._entries)
819
+
820
+
821
+ class Service:
822
+ """Process-wide state: the domain gate, the limiters, the upload/decrypt slots, the render slots and cache, and the
823
+ coalescing of refusals in docs.log."""
824
+
825
+ def __init__(self, domains=None, clock=time.time):
826
+ self.domains = tuple(domains) if domains else E.DEFAULT_DOMAINS
827
+ self.clock = clock
828
+ self._lock = threading.Lock()
829
+ self._uploads = {}
830
+ self._views = {} # login casefolded -> [window start, requests in it]
831
+ self._holders = set() # logins casefolded holding an upload slot (one each)
832
+ self._denied = {} # (word, source) -> [window start, refusals folded since, last login]
833
+ self._swept = float('-inf')
834
+ self.slots = threading.BoundedSemaphore(UPLOAD_SLOTS)
835
+ self.renders = threading.BoundedSemaphore(RENDER_SLOTS)
836
+ self.rendered = RenderCache()
837
+ self._pin_problem = None
838
+
839
+ @classmethod
840
+ def from_environ(cls, environ):
841
+ return cls(domains=E.parse_domains(environ.get('LANES_DOCS_DOMAINS') or environ.get('LANES_ENROL_DOMAINS')))
842
+
843
+ domain_ok = E.Service.domain_ok # enrold's rule over self.domains, the same function
844
+
845
+ def pinned(self):
846
+ """Whether the operator pinned request mode (enrold.read_mode): the proof the Serve header check ran."""
847
+ mode, problem = E.read_mode()
848
+ if problem and problem != self._pin_problem:
849
+ print(f'lanes-docsd: {problem}: every request is refused until `lanes enrol mode request` writes it again',
850
+ file=sys.stderr, flush=True)
851
+ self._pin_problem = problem
852
+ return mode == 'request'
853
+
854
+ def gate(self, headers, peer, method='GET'):
855
+ """(caller, None) when the request may be served, else (caller, the refusal's word). See the module docstring
856
+ for the order."""
857
+ caller = E.caller_of(headers)
858
+ if not E.through_serve(headers.get('X-Forwarded-For'), peer):
859
+ return caller, 'not-through-serve'
860
+ if not caller.present:
861
+ return caller, 'no-identity'
862
+ if not caller.login:
863
+ return caller, 'invalid-login'
864
+ if not self.pinned():
865
+ return caller, 'not-pinned'
866
+ if caller.login.casefold() not in E.member_logins():
867
+ return caller, 'not-a-member'
868
+ if not self.domain_ok(caller.login):
869
+ return caller, 'domain-not-allowed'
870
+ if not fetch_site_ok(headers, method):
871
+ return caller, 'cross-site'
872
+ return caller, None
873
+
874
+ def attempt(self, login):
875
+ """0 when `login` may upload now (the attempt is counted), else the seconds to wait."""
876
+ now, key = self.clock(), login.casefold()
877
+ with self._lock:
878
+ recent = [at for at in self._uploads.get(key, ()) if now - at < UPLOAD_WINDOW]
879
+ if len(recent) >= UPLOADS:
880
+ self._uploads[key] = recent
881
+ return int(recent[0] + UPLOAD_WINDOW - now) + 1
882
+ self._uploads[key] = recent + [now]
883
+ if len(self._uploads) > MAX_LOGINS:
884
+ for other in list(self._uploads):
885
+ kept = [at for at in self._uploads[other] if now - at < UPLOAD_WINDOW]
886
+ if kept:
887
+ self._uploads[other] = kept
888
+ else:
889
+ del self._uploads[other]
890
+ while len(self._uploads) > MAX_LOGINS:
891
+ self._uploads.pop(min(self._uploads, key=lambda name: self._uploads[name][-1]))
892
+ return 0
893
+
894
+ def view(self, login):
895
+ """(0, False) when `login` may make another request now (it is counted: VIEWS per VIEW_WINDOW s, a fixed
896
+ window), else (the seconds to wait, whether this is the window's first refusal — the one docs.log line)."""
897
+ now, key = self.clock(), login.casefold()
898
+ with self._lock:
899
+ window = self._views.get(key)
900
+ if window is None or now - window[0] >= VIEW_WINDOW:
901
+ if window is None and len(self._views) >= MAX_LOGINS:
902
+ for other in [name for name, (start, _n) in self._views.items() if now - start >= VIEW_WINDOW]:
903
+ del self._views[other]
904
+ while len(self._views) >= MAX_LOGINS:
905
+ self._views.pop(min(self._views, key=lambda name: self._views[name][0]))
906
+ window = self._views[key] = [now, 0]
907
+ window[1] += 1
908
+ if window[1] <= VIEWS:
909
+ return 0, False
910
+ return int(window[0] + VIEW_WINDOW - now) + 1, window[1] == VIEWS + 1
911
+
912
+ def take_upload_slot(self, login):
913
+ """Whether `login` got one of the UPLOAD_SLOTS (never a second one while it holds one: F5)."""
914
+ key = login.casefold()
915
+ with self._lock:
916
+ if key in self._holders or not self.slots.acquire(blocking=False):
917
+ return False
918
+ self._holders.add(key)
919
+ return True
920
+
921
+ def give_upload_slot(self, login):
922
+ with self._lock:
923
+ self._holders.discard(login.casefold())
924
+ self.slots.release()
925
+
926
+ def denied(self, login, word, source):
927
+ """Log a gate refusal, coalesced (F4): the first of a (word, source) is a `denied` line; more within
928
+ DENIED_REPEAT_S are counted, and flush_denied writes one `denied-repeat` line per window with the count."""
929
+ now = self.clock()
930
+ shown = login[:MAX_DENIED_LOGIN] if login else None
931
+ lines = self._sweep(now)
932
+ with self._lock:
933
+ entry = self._denied.get((word, source))
934
+ if entry is not None:
935
+ entry[1] += 1
936
+ entry[2] = shown
937
+ else:
938
+ while len(self._denied) >= MAX_DENIED_SOURCES: # forget the oldest; its count is said first
939
+ oldest = min(self._denied, key=lambda key: self._denied[key][0])
940
+ start, count, last = self._denied.pop(oldest)
941
+ if count:
942
+ lines.append((last, count, oldest[0]))
943
+ self._denied[(word, source)] = [now, 0, shown]
944
+ lines.append((shown, None, word))
945
+ for logged, count, said in lines:
946
+ D.audit('denied' if count is None else 'denied-repeat', logged, None, None, count, None, said, now)
947
+
948
+ def flush_denied(self, now=None):
949
+ """Write the `denied-repeat` lines whose window has passed (the pick-up's cycle calls this every few s)."""
950
+ now = self.clock() if now is None else now
951
+ for logged, count, said in self._sweep(now, force=True):
952
+ D.audit('denied-repeat', logged, None, None, count, None, said, now)
953
+
954
+ def _sweep(self, now, force=False):
955
+ """The repeat lines due at `now`, at most once a second unless `force`: a window with refusals folded into it
956
+ is said and restarted; one without is forgotten (the next refusal is a `denied` line again)."""
957
+ lines = []
958
+ with self._lock:
959
+ if not force and now - self._swept < 1:
960
+ return lines
961
+ self._swept = now
962
+ for key in list(self._denied):
963
+ start, count, last = self._denied[key]
964
+ if now - start < DENIED_REPEAT_S:
965
+ continue
966
+ if count:
967
+ lines.append((last, count, key[0]))
968
+ self._denied[key] = [now, 0, last]
969
+ else:
970
+ del self._denied[key]
971
+ return lines
972
+
973
+
974
+ # -------------------------------------------------------------------------------------------------------- HTTP
975
+
976
+ class _Reply(Exception):
977
+ def __init__(self, status, error, headers=()):
978
+ super().__init__(error)
979
+ self.status, self.error, self.headers = status, error, tuple(headers)
980
+
981
+
982
+ def resolve(path):
983
+ """(route, the id or file name it names, or None) for a request path, else (None, None)."""
984
+ for route, pattern in ROUTES:
985
+ found = pattern.fullmatch(path)
986
+ if found:
987
+ return route, (found.group(1) if found.groups() else None)
988
+ return None, None
989
+
990
+
991
+ class Handler(BaseHTTPRequestHandler):
992
+ server_version = 'lanes-docsd'
993
+ timeout = 30
994
+
995
+ def version_string(self):
996
+ return self.server_version
997
+
998
+ def log_message(self, format, *args): # no access log: docs.log is the record, the journal the errors
999
+ pass
1000
+
1001
+ def do_GET(self):
1002
+ self._route('GET')
1003
+
1004
+ def do_POST(self):
1005
+ self._route('POST')
1006
+
1007
+ def do_HEAD(self):
1008
+ self._route('HEAD')
1009
+
1010
+ def do_PUT(self):
1011
+ self._route('PUT')
1012
+
1013
+ def do_DELETE(self):
1014
+ self._route('DELETE')
1015
+
1016
+ def do_PATCH(self):
1017
+ self._route('PATCH')
1018
+
1019
+ def do_OPTIONS(self):
1020
+ self._route('OPTIONS')
1021
+
1022
+ # ------------------------------------------------------------------------------------------- plumbing
1023
+
1024
+ def _headers(self, status, content_type, length, headers, policy, cache):
1025
+ if self.command in ('POST', 'PUT', 'PATCH') and not self._body_read:
1026
+ self._discard() # an unread body would reset the client's connection (DEF-1854)
1027
+ self.send_response(status)
1028
+ self.send_header('Content-Type', content_type)
1029
+ self.send_header('Content-Length', str(length))
1030
+ self.send_header('Cache-Control', cache)
1031
+ self.send_header('X-Content-Type-Options', 'nosniff')
1032
+ self.send_header('X-Frame-Options', 'DENY')
1033
+ self.send_header('Referrer-Policy', 'no-referrer')
1034
+ self.send_header('Content-Security-Policy', policy)
1035
+ for name, value in headers:
1036
+ self.send_header(name, value)
1037
+ if self._discarded:
1038
+ self.send_header('Connection', 'close')
1039
+ self.end_headers()
1040
+
1041
+ def _send(self, status, body, content_type, headers=(), policy=DATA_POLICY, cache='no-store'):
1042
+ self._headers(status, content_type, len(body), headers, policy, cache)
1043
+ if self.command != 'HEAD':
1044
+ self.wfile.write(body)
1045
+ self._linger()
1046
+
1047
+ def _send_fd(self, fd, size, content_type, headers=(), policy=DATA_POLICY):
1048
+ self._headers(200, content_type, size, headers, policy, 'no-store')
1049
+ left = size
1050
+ while left > 0:
1051
+ chunk = os.read(fd, min(CHUNK, left))
1052
+ if not chunk:
1053
+ break # shrank under us: the connection closes short, never padded
1054
+ self.wfile.write(chunk)
1055
+ left -= len(chunk)
1056
+ if left:
1057
+ self.close_connection = True
1058
+
1059
+ def _page(self, status, body, headers=()):
1060
+ self._send(status, body, 'text/html; charset=utf-8', headers, PAGE_POLICY)
1061
+
1062
+ def _json(self, status, payload, headers=()):
1063
+ self._send(status, json.dumps(payload).encode('utf-8'), 'application/json; charset=utf-8', headers)
1064
+
1065
+ def _error(self, status, word, headers=()):
1066
+ if wants_html(self.headers):
1067
+ return self._page(status, message_page(status, word), headers)
1068
+ return self._json(status, {'error': word}, headers)
1069
+
1070
+ def _discard(self):
1071
+ """Read an unread request body away BEFORE a refusal is sent (DEF-1854): up to its Content-Length, at most
1072
+ DRAIN_LIMIT bytes. A client sends its whole body before it reads an answer; one answered and closed while it
1073
+ is still sending sees a reset or a broken pipe, never the refusal (W3 MEASURED that on the third of three
1074
+ concurrent 32 MiB uploads, when only 1 MiB was read away). What is left past DRAIN_LIMIT — or a chunked body,
1075
+ which is never read — is _linger's. The connection closes after the answer (`Connection: close`)."""
1076
+ self._body_read = self._discarded = True
1077
+ self.close_connection = True
1078
+ if self.headers.get('Transfer-Encoding') is not None:
1079
+ self._unread = DRAIN_LIMIT
1080
+ return
1081
+ try:
1082
+ declared = max(int(self.headers.get('Content-Length') or 0), 0)
1083
+ except ValueError:
1084
+ declared = 0
1085
+ drained, target = 0, min(declared, DRAIN_LIMIT)
1086
+ try:
1087
+ while drained < target:
1088
+ chunk = self.rfile.read(min(target - drained, CHUNK))
1089
+ if not chunk: # the client stopped sending: nothing more will come
1090
+ self._unread = 0
1091
+ return
1092
+ drained += len(chunk)
1093
+ except OSError:
1094
+ pass # a read timed out: what is left is _linger's
1095
+ self._unread = declared - drained
1096
+
1097
+ def _linger(self):
1098
+ """After a refusal's answer, when the body is still arriving (past DRAIN_LIMIT, chunked, or a read timed out):
1099
+ shut the write side — the client's next read sees the whole answer and then its end — and read the rest away
1100
+ for at most LINGER_S seconds and DRAIN_LIMIT bytes before the socket closes, so the close does not reset the
1101
+ connection under an answer the client has not read yet (DEF-1854)."""
1102
+ if not self._unread:
1103
+ return
1104
+ self._unread = 0
1105
+ try:
1106
+ self.wfile.flush()
1107
+ self.connection.shutdown(socket.SHUT_WR)
1108
+ deadline, left = time.monotonic() + LINGER_S, DRAIN_LIMIT
1109
+ while left > 0:
1110
+ remaining = deadline - time.monotonic()
1111
+ if remaining <= 0:
1112
+ break
1113
+ self.connection.settimeout(remaining)
1114
+ chunk = self.connection.recv(min(CHUNK, left))
1115
+ if not chunk:
1116
+ break
1117
+ left -= len(chunk)
1118
+ except OSError:
1119
+ pass
1120
+
1121
+ def _body(self):
1122
+ if self.headers.get('Transfer-Encoding') is not None:
1123
+ self._discard() # chunked bodies are not read: answer, then _linger
1124
+ raise _Reply(411, 'length-required')
1125
+ length = self.headers.get('Content-Length')
1126
+ if length is None:
1127
+ raise _Reply(411, 'length-required')
1128
+ try:
1129
+ size = int(length)
1130
+ except ValueError:
1131
+ size = -1
1132
+ if size < 0:
1133
+ self._discard()
1134
+ raise _Reply(400, 'bad-request')
1135
+ if size > MAX_BODY:
1136
+ self._discard()
1137
+ raise _Reply(413, 'too-large')
1138
+ # The whole body has a deadline, not just each read (F5): a sender that trickles a byte every few seconds held
1139
+ # an upload slot for as long as it liked. Read into one buffer of the declared size (no second copy).
1140
+ data, got = bytearray(size), 0
1141
+ view = memoryview(data)
1142
+ deadline = time.monotonic() + BODY_GRACE_S + size / BODY_MIN_RATE
1143
+ try:
1144
+ while got < size:
1145
+ left = deadline - time.monotonic()
1146
+ if left <= 0:
1147
+ raise TimeoutError('the body did not arrive in time')
1148
+ self.connection.settimeout(min(left, self.timeout))
1149
+ read = self.rfile.readinto1(view[got:got + CHUNK])
1150
+ if not read:
1151
+ break
1152
+ got += read
1153
+ except TimeoutError:
1154
+ self._body_read = self._discarded = True # the rest is not waited for: answer and close
1155
+ self._unread = 0
1156
+ self.close_connection = True
1157
+ raise _Reply(408, 'request-timeout') from None
1158
+ except OSError:
1159
+ self._body_read = self._discarded = True # the client went away
1160
+ self._unread = 0
1161
+ self.close_connection = True
1162
+ raise _Reply(400, 'bad-request') from None
1163
+ finally:
1164
+ try:
1165
+ self.connection.settimeout(self.timeout)
1166
+ except OSError:
1167
+ pass
1168
+ self._body_read = True
1169
+ if got != size:
1170
+ self.close_connection = True
1171
+ raise _Reply(400, 'bad-request')
1172
+ return data
1173
+
1174
+ # --------------------------------------------------------------------------------------------- routes
1175
+
1176
+ def _route(self, method):
1177
+ self._body_read = self._discarded = False
1178
+ self._unread = 0
1179
+ service = self.server.service
1180
+ try:
1181
+ caller, refusal = service.gate(self.headers, self.client_address[0], method)
1182
+ if refusal:
1183
+ service.denied(caller.login, refusal,
1184
+ E.source_key(self.headers.get('X-Forwarded-For'), self.client_address[0]))
1185
+ return self._error(403, refusal)
1186
+ route, name = resolve(urllib.parse.urlsplit(self.path).path)
1187
+ if route != 'up': # every request but an upload counts against the login (F2)
1188
+ wait, first = service.view(caller.login)
1189
+ if wait:
1190
+ if first: # one line per login per window: `view … rate-limited`
1191
+ D.audit('view', caller.login, None, None, None, None, 'rate-limited')
1192
+ raise _Reply(429, 'rate-limited', (('Retry-After', str(wait)),))
1193
+ if route is None:
1194
+ return self._error(404, 'not-found')
1195
+ allowed = METHODS.get(route, ('GET',))
1196
+ if method not in allowed:
1197
+ return self._error(405, 'method-not-allowed', (('Allow', ', '.join(allowed)),))
1198
+ if route == 'static':
1199
+ return self._static(name)
1200
+ if route == 'up':
1201
+ return self._upload(service, caller)
1202
+ workspaces = member_workspaces(caller.login)
1203
+ if route == 'index':
1204
+ reg = registry()
1205
+ docs, targets = visible_docs(caller.login, workspaces), upload_targets(workspaces, reg)
1206
+ ws, lane = index_filter(urllib.parse.urlsplit(self.path).query, workspaces, docs, targets)
1207
+ return self._page(200, index_page(caller.login, docs, reg, targets, ws, lane))
1208
+ return self._document(route, name, caller.login, workspaces)
1209
+ except _Reply as reply:
1210
+ try:
1211
+ return self._error(reply.status, reply.error, reply.headers)
1212
+ except OSError: # the client went away before its answer (a stalled upload's)
1213
+ self.close_connection = True
1214
+ except Exception: # noqa: BLE001 — one request's bug never stops the service
1215
+ traceback.print_exc(file=sys.stderr)
1216
+ try:
1217
+ self._error(500, 'server-error')
1218
+ except OSError:
1219
+ pass
1220
+
1221
+ def _static(self, name):
1222
+ try:
1223
+ directory = L.open_real_dir(web_dir(), ())
1224
+ except OSError:
1225
+ return self._error(404, 'not-found')
1226
+ try:
1227
+ data = L.read_regular_at(directory, name, MAX_STATIC)
1228
+ finally:
1229
+ os.close(directory)
1230
+ if data is None:
1231
+ return self._error(404, 'not-found')
1232
+ return self._send(200, data, STATIC[name], cache='no-cache')
1233
+
1234
+ def _document(self, route, doc_id, login, workspaces):
1235
+ try:
1236
+ found = find_doc(doc_id, login, workspaces)
1237
+ except D.DocsError:
1238
+ found = None
1239
+ if found is None or (route == 'artifact' and found[0]['kind'] not in ARTIFACT_TYPES):
1240
+ D.audit(route, login, doc_id, found[0]['lane'] if found else None, None, None, 'not-found')
1241
+ return self._error(404, 'not-found')
1242
+ meta, parts = found
1243
+ if route == 'receipt':
1244
+ data = read_receipt(parts)
1245
+ if data is None:
1246
+ D.audit(route, login, doc_id, meta['lane'], None, None, 'not-found')
1247
+ return self._error(404, 'not-found')
1248
+ D.audit(route, login, doc_id, meta['lane'], meta['bytes'], meta['sha256'], 'ok')
1249
+ return self._send(200, data, 'application/json; charset=utf-8')
1250
+ try:
1251
+ fd = open_doc(meta, parts)
1252
+ except (OSError, L.LanesError):
1253
+ D.audit(route, login, doc_id, meta['lane'], None, None, 'not-found')
1254
+ return self._error(404, 'not-found')
1255
+ try:
1256
+ if meta.get('enc'):
1257
+ return self._encrypted(route, meta, fd, login)
1258
+ if route == 'view':
1259
+ body = None
1260
+ if meta['kind'] == 'md' and meta['bytes'] <= mdlite.MAX_BYTES:
1261
+ body = self._rendered(route, meta, login, lambda: read_all(fd, mdlite.MAX_BYTES))
1262
+ D.audit(route, login, doc_id, meta['lane'], meta['bytes'], meta['sha256'], 'ok')
1263
+ return self._page(200, view_page(meta, body))
1264
+ D.audit(route, login, doc_id, meta['lane'], meta['bytes'], meta['sha256'], 'ok')
1265
+ if route == 'raw':
1266
+ disposition = f'attachment; filename="{download_name(meta["name"])}"'
1267
+ return self._send_fd(fd, meta['bytes'], content_type_of(meta),
1268
+ (('Content-Disposition', disposition), (SHA_HEADER, meta['sha256'])))
1269
+ return self._send_fd(fd, meta['bytes'], ARTIFACT_TYPES[meta['kind']], ((SHA_HEADER, meta['sha256']),),
1270
+ ARTIFACT_POLICY)
1271
+ finally:
1272
+ os.close(fd)
1273
+
1274
+ def _rendered(self, route, meta, login, source):
1275
+ """The Markdown document `meta` rendered (None: too large once decoded), from the service's cache by its sha256,
1276
+ else rendered from source()'s bytes in one of RENDER_SLOTS (waiting at most RENDER_WAIT_S, else 503 busy) and
1277
+ kept (F2: a hostile 1 MiB document renders for seconds, and a page could ask for it in a loop)."""
1278
+ service = self.server.service
1279
+ hit, body = service.rendered.get(meta['sha256'])
1280
+ if hit:
1281
+ return body
1282
+ if not service.renders.acquire(timeout=RENDER_WAIT_S):
1283
+ D.audit(route, login, meta['id'], meta['lane'], None, None, 'busy')
1284
+ raise _Reply(503, 'busy', (('Retry-After', str(BUSY_RETRY)),))
1285
+ try:
1286
+ try: # invalid UTF-8 becomes U+FFFD, which can push it past the limit
1287
+ body = mdlite.render(bytes(source()).decode('utf-8', 'replace'))
1288
+ except mdlite.TooLarge:
1289
+ body = None
1290
+ finally:
1291
+ service.renders.release()
1292
+ service.rendered.put(meta['sha256'], body)
1293
+ return body
1294
+
1295
+ def _decrypted(self, route, meta, fd, identity, login):
1296
+ """The plaintext of an encrypted document (docs_store.decrypt, checked against the receipt), one at a time
1297
+ (_DECRYPTS) and in one of the UPLOAD_SLOTS big buffers it shares with uploads (F8), each waited for at most
1298
+ DECRYPT_WAIT_S, else 503 busy."""
1299
+ service = self.server.service
1300
+ if not _DECRYPTS.acquire(timeout=DECRYPT_WAIT_S):
1301
+ D.audit(route, login, meta['id'], meta['lane'], None, None, 'busy')
1302
+ raise _Reply(503, 'busy', (('Retry-After', str(BUSY_RETRY)),))
1303
+ try:
1304
+ if not service.slots.acquire(timeout=DECRYPT_WAIT_S):
1305
+ D.audit(route, login, meta['id'], meta['lane'], None, None, 'busy')
1306
+ raise _Reply(503, 'busy', (('Retry-After', str(BUSY_RETRY)),))
1307
+ try:
1308
+ return D.decrypt(meta, fd, identity)
1309
+ except D.EncryptError:
1310
+ D.audit(route, login, meta['id'], meta['lane'], None, None, 'encrypt-failed')
1311
+ raise _Reply(500, 'server-error') from None
1312
+ finally:
1313
+ service.slots.release()
1314
+ finally:
1315
+ _DECRYPTS.release()
1316
+
1317
+ def _encrypted(self, route, meta, fd, login):
1318
+ """An encrypted document (docs_store's `age` switch, meta `enc`; fd its stored `<name>.age`). With an identity
1319
+ (LANES_DOCS_AGE_IDENTITY) and at most MAX_BODY bytes it is decrypted on the fly (_decrypted); raw and artifact
1320
+ then answer as for a plain document, and a view renders it unless the render cache already holds it. Without
1321
+ one, raw and artifact are 409 encrypted (the page names uploads/), and the view shows the facts only."""
1322
+ try:
1323
+ identity = D.age_settings()[1]
1324
+ except D.DocsError:
1325
+ identity = None
1326
+ if route == 'view':
1327
+ body = None
1328
+ if identity and meta['kind'] == 'md' and meta['bytes'] <= mdlite.MAX_BYTES:
1329
+ body = self._rendered(route, meta, login, lambda: self._decrypted(route, meta, fd, identity, login))
1330
+ D.audit(route, login, meta['id'], meta['lane'], meta['bytes'], meta['sha256'], 'ok')
1331
+ return self._page(200, view_page(meta, body, locked=not identity))
1332
+ data = None
1333
+ if identity and meta['bytes'] <= MAX_BODY:
1334
+ data = self._decrypted(route, meta, fd, identity, login)
1335
+ if data is None:
1336
+ D.audit(route, login, meta['id'], meta['lane'], None, None, 'encrypted')
1337
+ raise _Reply(409, 'encrypted')
1338
+ D.audit(route, login, meta['id'], meta['lane'], meta['bytes'], meta['sha256'], 'ok')
1339
+ if route == 'raw':
1340
+ disposition = f'attachment; filename="{download_name(meta["name"])}"'
1341
+ return self._send(200, data, content_type_of(meta),
1342
+ (('Content-Disposition', disposition), (SHA_HEADER, meta['sha256'])))
1343
+ return self._send(200, data, ARTIFACT_TYPES[meta['kind']], ((SHA_HEADER, meta['sha256']),), ARTIFACT_POLICY)
1344
+
1345
+ def _upload(self, service, caller):
1346
+ login = caller.login # the gate refused one from another site already (F2)
1347
+ wait = service.attempt(login)
1348
+ if wait:
1349
+ D.audit('upload', login, None, None, None, None, 'rate-limited')
1350
+ raise _Reply(429, 'rate-limited', (('Retry-After', str(wait)),))
1351
+ if not service.take_upload_slot(login):
1352
+ D.audit('upload', login, None, None, None, None, 'busy')
1353
+ raise _Reply(503, 'busy', (('Retry-After', str(BUSY_RETRY)),))
1354
+ try:
1355
+ return self._upload_in_slot(login)
1356
+ finally:
1357
+ service.give_upload_slot(login)
1358
+
1359
+ def _upload_in_slot(self, login):
1360
+ boundary = form_boundary(self.headers.get('Content-Type'))
1361
+ if boundary is None:
1362
+ D.audit('upload', login, None, None, None, None, 'unsupported-media-type')
1363
+ raise _Reply(415, 'unsupported-media-type')
1364
+ try:
1365
+ data = self._body()
1366
+ except _Reply as reply:
1367
+ D.audit('upload', login, None, None, None, None, reply.error)
1368
+ raise
1369
+ fields = {}
1370
+ reg = registry()
1371
+ try:
1372
+ fields, files = parse_form(data, boundary)
1373
+ if not files:
1374
+ raise FormError('no-file')
1375
+ ws, lane = upload_target(fields, member_workspaces(login), reg)
1376
+ except FormError as error:
1377
+ lane = fields.get('lane')
1378
+ D.audit('upload', login, None, lane if isinstance(lane, str) and L.IDENT.match(lane) else None, None, None,
1379
+ error.error)
1380
+ raise _Reply(503 if error.error == 'registry-unavailable' else 400, error.error) from None
1381
+ workspace = next(entry for entry in reg['workspaces'] if isinstance(entry, dict) and entry.get('id') == ws)
1382
+ try: # where the files land: uploads/ of the workspace (W5)
1383
+ folder, home, parts = U.landing(reg, workspace)
1384
+ uploads = U.open_uploads(home, parts)
1385
+ except (L.LanesError, OSError):
1386
+ D.audit('upload', login, None, lane, None, None, 'uploads-unavailable')
1387
+ raise _Reply(503, 'uploads-unavailable') from None
1388
+ receipts, failure = [], None
1389
+ try:
1390
+ for name, start, end in files:
1391
+ try:
1392
+ meta, landed = U.receive(uploads, D.clean_name(name), _Slice(data, start, end), login=login, ws=ws,
1393
+ lane=lane, source='web')
1394
+ except U.UploadError as error: # intake logged it; nothing of this file landed
1395
+ failure = {'busy': (503, 'busy'), 'encrypt-failed': (500, 'encrypt-failed')}.get(
1396
+ error.word, (400, 'bad-request'))
1397
+ break
1398
+ except (OSError, L.LanesError): # could not land: the store document was removed again
1399
+ D.audit('upload', login, None, lane, None, None, 'uploads-unavailable')
1400
+ failure = (503, 'uploads-unavailable')
1401
+ break
1402
+ receipts.append({'id': meta['id'], 'name': meta['name'], 'sha256': meta['sha256'],
1403
+ 'bytes': meta['bytes'], 'url': D.URL.format(meta['id']), 'at': meta['at'],
1404
+ 'landed': f'{folder}/uploads/{landed}'})
1405
+ finally:
1406
+ os.close(uploads)
1407
+ status, error = failure if failure else (200, None)
1408
+ headers = (('Retry-After', str(BUSY_RETRY)),) if status == 503 else ()
1409
+ if wants_html(self.headers):
1410
+ return self._page(status, receipts_page(receipts, error), headers)
1411
+ payload = {'receipts': receipts}
1412
+ if error:
1413
+ payload['error'] = error
1414
+ return self._json(status, payload, headers)
1415
+
1416
+
1417
+ class Server(ThreadingHTTPServer):
1418
+ daemon_threads = True
1419
+ request_queue_size = 32
1420
+
1421
+ def __init__(self, address, service):
1422
+ self.service = service
1423
+ super().__init__(address, Handler)
1424
+
1425
+ def server_bind(self): # HTTPServer's getfqdn() lookup is not needed on loopback
1426
+ socketserver.TCPServer.server_bind(self)
1427
+ self.server_name, self.server_port = self.server_address[:2]
1428
+
1429
+
1430
+ # ------------------------------------------------------------------------------------------------ publish pick-up
1431
+
1432
+ PICKUP_INTERVAL = 5 # seconds between two pick-up cycles (lane_core.PUBLISH_LATENCY_S)
1433
+ PUBLISH_LIMIT = MAX_BODY # a published file, at most: the docs limit (lane_core.PUBLISH_MAX)
1434
+ MAX_REQUEST = 16 * 1024 # one request's JSON
1435
+ MAX_PATH = 4096 # a request's `path`
1436
+ MAX_PICKUPS = 16 # requests taken from one container per cycle; the rest wait for the next
1437
+ MAX_RECEIPTS = 16 # receipts kept per lane in a container, the newest
1438
+ PUBLISHES = 30 # requests taken per container in PUBLISH_WINDOW seconds; more are refused
1439
+ PUBLISH_WINDOW = 600
1440
+ PUBLISHED_PUSH_S = 60 # one `published` push per lane in this many seconds, at most
1441
+ PUBLISH_KINDS = ('md', 'html', 'svg', 'file') # lane_core.PUBLISH_KINDS: docs_store.KINDS - upload
1442
+ PUBLISH_DIR = tuple(L.CONTAINER_HOME.strip('/').split('/')) + ('.lz', 'state', 'publish')
1443
+ HOME_PARTS = tuple(L.CONTAINER_HOME.strip('/').split('/'))
1444
+ REQUEST_SUFFIX, RECEIPT_SUFFIX, CLAIM_SUFFIX = '.json', '.receipt.json', '.taking'
1445
+ MAX_ENTRIES = 4096 # directory entries surveyed per container per cycle; more: refused (F1)
1446
+ OVER_CAP_SAY_S = 3600 # an over-cap container is said in the journal once per this many seconds
1447
+ REFUSALS = ('unreadable-request', 'bad-version', 'bad-request', 'bad-kind', 'bad-title', 'too-large', 'unknown-lane',
1448
+ 'rate-limited', 'for-not-a-member', 'no-recipient', 'outside-home', 'not-a-file', 'sha256-mismatch',
1449
+ 'invalid', 'error') # the words docs.log says; a receipt says receipt_reason()'s
1450
+ NOT_DELIVERED = ('for-not-a-member', 'no-recipient') # their receipt says `not-delivered` (F6)
1451
+ _PLANTED = (errno.EISDIR, errno.ENOTEMPTY, errno.EEXIST) # a rename onto a directory planted at its target
1452
+ _UNCLAIMED = object() # _take: the request could not be claimed; the next one is tried
1453
+ _SHA256 = re.compile(r'[0-9a-f]{64}', re.ASCII)
1454
+ _CHILD_DIR = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | getattr(os, 'O_CLOEXEC', 0)
1455
+ _LEAF = os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK | getattr(os, 'O_CLOEXEC', 0)
1456
+
1457
+
1458
+ class Refused(Exception):
1459
+ """A publish request refused: `reason` (one of REFUSALS) goes into its receipt, with the host's sha256 and size of
1460
+ the file when they are known."""
1461
+
1462
+ def __init__(self, reason, sha256=None, size=None):
1463
+ super().__init__(reason)
1464
+ self.reason, self.sha256, self.size = reason, sha256, size
1465
+
1466
+
1467
+ def container_roots():
1468
+ """[(container, root)] for every container containers.json names (`lanes sync`, statusd: L.refresh_containers)
1469
+ whose init pid is alive: root = /proc/<pid>/root. A pid whose root is the host's own `/` is skipped — a container
1470
+ that went away and a host process that got its pid — so the host's home is never read as a container's. And a pid
1471
+ whose /home/coder (opened without following a link) is not the home volume containers.json records for the
1472
+ container (`home`, the host path docker reports; compared by st_dev and st_ino, a mount shows its source's) is
1473
+ skipped too: another container got the pid, and its requests would be taken as this container's lanes' (F10). When
1474
+ this account cannot see the recorded volume, or none is recorded, only the host-root check applies (DEF-1918)."""
1475
+ info = L.read_json(L.STATE / 'containers.json', {}) or {}
1476
+ try:
1477
+ host = os.stat('/')
1478
+ except OSError:
1479
+ host = None
1480
+ found = []
1481
+ for name in sorted(info if isinstance(info, dict) else ()):
1482
+ entry = info[name]
1483
+ pid = entry.get('pid') if isinstance(entry, dict) else None
1484
+ if not L.IDENT.match(name) or not isinstance(pid, int) or isinstance(pid, bool) or pid <= 0:
1485
+ continue
1486
+ root = Path(L.proc_root()) / str(pid) / 'root'
1487
+ try:
1488
+ seen = os.stat(root)
1489
+ except OSError:
1490
+ continue
1491
+ if host is not None and (seen.st_dev, seen.st_ino) == (host.st_dev, host.st_ino):
1492
+ continue
1493
+ if not _home_matches(root, entry.get('home')):
1494
+ continue
1495
+ found.append((name, root))
1496
+ return found
1497
+
1498
+
1499
+ def _home_matches(root, home):
1500
+ """False only when both the recorded home volume `home` and the container's /home/coder can be seen and they are
1501
+ not the same directory."""
1502
+ if not isinstance(home, str) or not home.startswith('/'):
1503
+ return True
1504
+ try:
1505
+ volume = os.stat(home)
1506
+ fd = L.open_real_dir(root, HOME_PARTS)
1507
+ except (OSError, L.LanesError):
1508
+ return True
1509
+ try:
1510
+ mounted = os.fstat(fd)
1511
+ except OSError:
1512
+ return True
1513
+ finally:
1514
+ os.close(fd)
1515
+ return (volume.st_dev, volume.st_ino) == (mounted.st_dev, mounted.st_ino)
1516
+
1517
+
1518
+ def lane_places(reg):
1519
+ """{lane id: (workspace id, container)} of every registry lane that runs in a container."""
1520
+ places = {}
1521
+ for workspace in (reg or {}).get('workspaces') or []:
1522
+ if not isinstance(workspace, dict) or not isinstance(workspace.get('id'), str):
1523
+ continue
1524
+ for lane in workspace.get('lanes') or []:
1525
+ if isinstance(lane, dict) and isinstance(lane.get('id'), str) and isinstance(lane.get('container'), str):
1526
+ places.setdefault(lane['id'], (workspace['id'], lane['container']))
1527
+ return places
1528
+
1529
+
1530
+ def read_request(directory, name):
1531
+ """The checked request in `name` of the open lane directory -> {path, kind, claimed_sha256, bytes, title, for}.
1532
+ Raises Refused (see the module docstring, step 2)."""
1533
+ raw = L.read_regular_at(directory, name, MAX_REQUEST)
1534
+ try:
1535
+ request = json.loads(raw.decode('utf-8')) if raw is not None else None
1536
+ except ValueError:
1537
+ request = None
1538
+ if not isinstance(request, dict):
1539
+ raise Refused('unreadable-request')
1540
+ if request.get('v') != 1:
1541
+ raise Refused('bad-version')
1542
+ path, sha256, size = request.get('path'), request.get('claimed_sha256'), request.get('bytes')
1543
+ title, for_login, mime = request.get('title'), request.get('for'), request.get('mime')
1544
+ if not (isinstance(path, str) and path.startswith('/') and '\x00' not in path and len(path) <= MAX_PATH
1545
+ and isinstance(sha256, str) and _SHA256.fullmatch(sha256)
1546
+ and isinstance(size, int) and not isinstance(size, bool) and size >= 0
1547
+ and all(value is None or isinstance(value, str) for value in (title, for_login, mime))):
1548
+ raise Refused('bad-request')
1549
+ if request.get('kind') not in PUBLISH_KINDS:
1550
+ raise Refused('bad-kind')
1551
+ try:
1552
+ title = D._title(title)
1553
+ except L.LanesError:
1554
+ raise Refused('bad-title') from None
1555
+ if size > PUBLISH_LIMIT:
1556
+ raise Refused('too-large')
1557
+ return {'path': path, 'kind': request['kind'], 'claimed_sha256': sha256, 'bytes': size, 'title': title,
1558
+ 'for': for_login or None}
1559
+
1560
+
1561
+ def recipient(lane, ws, for_login, grants, profiles):
1562
+ """(store login, `for` list) of a document published from `lane` in `ws` (the module docstring, "Who a published
1563
+ document is for"). `grants` is member_grants(), `profiles` users.load_all()'s. Raises Refused."""
1564
+ if for_login is not None:
1565
+ entry = grants.get(for_login.casefold())
1566
+ if entry is None:
1567
+ raise Refused('for-not-a-member')
1568
+ return entry[0], [entry[0]]
1569
+ opened = [(profile['last_opened']['at'], profile['login']) for profile in profiles
1570
+ if isinstance(profile.get('last_opened'), dict) and profile['last_opened'].get('lane') == lane
1571
+ and profile['login'].casefold() in grants]
1572
+ if opened:
1573
+ login = grants[max(opened)[1].casefold()][0]
1574
+ return login, [login]
1575
+ listing = sorted((spelling for spelling, granted in grants.values() if ws in granted), key=str.casefold)
1576
+ if not listing:
1577
+ raise Refused('no-recipient')
1578
+ return listing[0], []
1579
+
1580
+
1581
+ def open_published(root, path):
1582
+ """(file name, read-only fd) of the file a request names, in the container whose root directory is `root` (the
1583
+ module docstring, step 3). The caller closes the fd. Raises Refused: outside-home, not-a-file, too-large."""
1584
+ try:
1585
+ real = L.container_realpath(root, path)
1586
+ except L.LanesError:
1587
+ raise Refused('not-a-file') from None
1588
+ parts = tuple(part for part in real.split('/') if part)
1589
+ if len(parts) <= len(HOME_PARTS) or parts[:len(HOME_PARTS)] != HOME_PARTS:
1590
+ raise Refused('outside-home')
1591
+ try:
1592
+ directory = L.open_real_dir(root, parts[:-1])
1593
+ except (OSError, L.LanesError):
1594
+ raise Refused('not-a-file') from None
1595
+ try:
1596
+ fd = os.open(parts[-1], _LEAF, dir_fd=directory)
1597
+ except OSError:
1598
+ raise Refused('not-a-file') from None
1599
+ finally:
1600
+ os.close(directory)
1601
+ try:
1602
+ info = os.fstat(fd)
1603
+ if not stat.S_ISREG(info.st_mode):
1604
+ raise Refused('not-a-file')
1605
+ if info.st_size > PUBLISH_LIMIT:
1606
+ raise Refused('too-large', size=info.st_size)
1607
+ except BaseException:
1608
+ os.close(fd)
1609
+ raise
1610
+ return parts[-1], fd
1611
+
1612
+
1613
+ class _Hashing:
1614
+ """docs_store.intake's reader over the published file: hashes what it hands on and, at the end of the file, raises
1615
+ Refused('sha256-mismatch') for a sha256 other than the claimed one — inside intake, before anything is committed,
1616
+ so intake removes what it wrote."""
1617
+
1618
+ def __init__(self, fd, claimed):
1619
+ self.fd, self.claimed, self.digest, self.size = fd, claimed, hashlib.sha256(), 0
1620
+
1621
+ def read(self, size=-1):
1622
+ chunk = os.read(self.fd, size if isinstance(size, int) and size > 0 else CHUNK)
1623
+ if chunk:
1624
+ self.digest.update(chunk)
1625
+ self.size += len(chunk)
1626
+ elif self.digest.hexdigest() != self.claimed:
1627
+ raise Refused('sha256-mismatch', self.digest.hexdigest(), self.size)
1628
+ return chunk
1629
+
1630
+
1631
+ def receipt_reason(reason):
1632
+ """The word a receipt says for a refusal: docs.log's, except `not-delivered` for both recipient refusals (F6)."""
1633
+ return 'not-delivered' if reason in NOT_DELIVERED else reason
1634
+
1635
+
1636
+ def receipt_of(request_id, now, meta=None, refusal=None):
1637
+ """The receipt written back for a request (DM-LANES-PUBLISH-RECEIPT): `meta` of the stored document, or a
1638
+ Refused."""
1639
+ if meta is not None:
1640
+ return {'v': 1, 'id': meta['id'], 'request': request_id, 'sha256': meta['sha256'], 'bytes': meta['bytes'],
1641
+ 'url': D.URL.format(meta['id']), 'at': meta['at'], 'outcome': 'ok'}
1642
+ return {'v': 1, 'id': None, 'request': request_id, 'sha256': refusal.sha256, 'bytes': refusal.size, 'url': None,
1643
+ 'at': D.stamp(now), 'outcome': 'refused', 'reason': receipt_reason(refusal.reason)}
1644
+
1645
+
1646
+ def prune_receipts(directory, names=None):
1647
+ """Keep the newest MAX_RECEIPTS receipts of one lane directory (ids sort by their base36 time). `names`: the
1648
+ receipt names the survey saw plus those written since; else the directory is read, at most MAX_ENTRIES entries."""
1649
+ if names is None:
1650
+ names = []
1651
+ try:
1652
+ with os.scandir(directory) as entries:
1653
+ for count, entry in enumerate(entries):
1654
+ if count >= MAX_ENTRIES:
1655
+ return
1656
+ names.append(entry.name)
1657
+ except OSError:
1658
+ return
1659
+ receipts = sorted(name for name in set(names) if name.endswith(RECEIPT_SUFFIX)
1660
+ and D.ID_RE.fullmatch(name[:-len(RECEIPT_SUFFIX)]))
1661
+ for name in receipts[:-MAX_RECEIPTS]:
1662
+ try:
1663
+ os.unlink(name, dir_fd=directory)
1664
+ except OSError:
1665
+ pass
1666
+
1667
+
1668
+ def survey(publish):
1669
+ """[(lane, pending request ids oldest first, receipt names)] of the open publish directory `publish`, lanes in name
1670
+ order — or None when its directory and its lane directories hold more than MAX_ENTRIES entries between them: the
1671
+ container is refused for this cycle (F1). Read with os.scandir and never past MAX_ENTRIES entries in all, keeping
1672
+ only the names that matter (a lane-id directory; a request or receipt name), so neither time nor memory follows
1673
+ what a container plants."""
1674
+ seen, lanes = 0, []
1675
+ with os.scandir(publish) as entries:
1676
+ for entry in entries:
1677
+ seen += 1
1678
+ if seen > MAX_ENTRIES:
1679
+ return None
1680
+ name = entry.name
1681
+ if not L.IDENT.match(name) or len(name) > users.MAX_LANE_ID:
1682
+ continue
1683
+ try:
1684
+ if entry.is_dir(follow_symlinks=False):
1685
+ lanes.append(name)
1686
+ except OSError:
1687
+ continue
1688
+ found = []
1689
+ for lane in sorted(lanes):
1690
+ try:
1691
+ directory = os.open(lane, _CHILD_DIR, dir_fd=publish)
1692
+ except OSError:
1693
+ continue # a link or a file where a lane directory would be
1694
+ pending, receipts = [], []
1695
+ try:
1696
+ with os.scandir(directory) as entries:
1697
+ for entry in entries:
1698
+ seen += 1
1699
+ if seen > MAX_ENTRIES:
1700
+ return None
1701
+ name = entry.name
1702
+ if name.endswith(RECEIPT_SUFFIX):
1703
+ if D.ID_RE.fullmatch(name[:-len(RECEIPT_SUFFIX)]):
1704
+ receipts.append(name)
1705
+ elif name.endswith(REQUEST_SUFFIX) and D.ID_RE.fullmatch(name[:-len(REQUEST_SUFFIX)]):
1706
+ pending.append(name[:-len(REQUEST_SUFFIX)])
1707
+ except OSError:
1708
+ continue
1709
+ finally:
1710
+ os.close(directory)
1711
+ found.append((lane, sorted(pending), receipts))
1712
+ return found
1713
+
1714
+
1715
+ def _plain(error):
1716
+ """An OSError without its file names, so one lane's repeated failure is one journal line, not one per request."""
1717
+ return type(error)(error.errno, error.strerror) if isinstance(error, OSError) and error.errno else error
1718
+
1719
+
1720
+ def clear_planted(directory, name):
1721
+ """Make `name` in the open container directory free for a rename onto it when a directory was planted there: an
1722
+ empty one is removed, any other moved aside under a new dot name (never followed, never read; the host deletes
1723
+ nothing of the container's but its own claims). -> whether the name now holds no directory."""
1724
+ try:
1725
+ info = os.stat(name, dir_fd=directory, follow_symlinks=False)
1726
+ except FileNotFoundError:
1727
+ return True
1728
+ except OSError:
1729
+ return False
1730
+ if not stat.S_ISDIR(info.st_mode):
1731
+ return True
1732
+ try:
1733
+ os.rmdir(name, dir_fd=directory)
1734
+ return True
1735
+ except OSError:
1736
+ pass
1737
+ try:
1738
+ os.rename(name, f'.{name}.{os.urandom(4).hex()}.planted', src_dir_fd=directory, dst_dir_fd=directory)
1739
+ return True
1740
+ except OSError:
1741
+ return False
1742
+
1743
+
1744
+ class Pickup:
1745
+ """The publish pick-up (the module docstring): one cycle() per PICKUP_INTERVAL in run(), on main()'s thread."""
1746
+
1747
+ def __init__(self, clock=time.time, notify_conf=None, opener=None, tick=None):
1748
+ self.clock, self.notify_conf, self.opener, self.tick = clock, notify_conf, opener, tick
1749
+ self.errors = {} # where -> the last error said there (each distinct one is logged once)
1750
+ self.taken = {} # container -> the times requests were taken (PUBLISHES per PUBLISH_WINDOW)
1751
+ self.pushed = {} # lane -> when its last `published` push was tried
1752
+ self.over_cap = {} # container -> when its over-cap refusal was last said (OVER_CAP_SAY_S)
1753
+
1754
+ def run(self, stop=None):
1755
+ stop = stop or threading.Event()
1756
+ while not stop.is_set():
1757
+ started = time.monotonic()
1758
+ try:
1759
+ self.cycle()
1760
+ except Exception as error: # noqa: BLE001 — one bad cycle never ends the pick-up
1761
+ self.error('cycle', error)
1762
+ if self.tick is not None: # main(): the service's docs.log refusal lines that are due (F4)
1763
+ try:
1764
+ self.tick()
1765
+ except Exception as error: # noqa: BLE001
1766
+ self.error('tick', error)
1767
+ stop.wait(max(0.5, PICKUP_INTERVAL - (time.monotonic() - started)))
1768
+
1769
+ def error(self, where, error):
1770
+ message = f'{type(error).__name__}: {error}'
1771
+ if self.errors.get(where) != message:
1772
+ self.errors[where] = message
1773
+ print(f'lanes-docsd: pick-up {where}: {message}', file=sys.stderr, flush=True)
1774
+
1775
+ def cycle(self):
1776
+ """One pass over every container -> [(container, lane, request id, outcome)] for the requests taken. Without a
1777
+ registry nothing is taken: no lane can be placed, and a request waits rather than being refused."""
1778
+ reg = registry()
1779
+ if reg is None:
1780
+ return []
1781
+ places, context, published, results = lane_places(reg), {'reg': reg}, [], []
1782
+ for container, root in container_roots():
1783
+ try:
1784
+ results += self._container(container, root, places, context, published)
1785
+ except Exception as error: # noqa: BLE001 — one container's trouble never costs the others
1786
+ self.error(container, error)
1787
+ names = N.lane_names(reg) if published else {}
1788
+ for lane, meta in published:
1789
+ self._push(lane, meta, names)
1790
+ return results
1791
+
1792
+ def _container(self, container, root, places, context, published):
1793
+ try:
1794
+ publish = L.open_real_dir(root, PUBLISH_DIR)
1795
+ except FileNotFoundError:
1796
+ return [] # nothing was ever published from this container
1797
+ except (OSError, L.LanesError) as error:
1798
+ self.error(container, error) # a component that is a link (or not a directory): never followed
1799
+ return []
1800
+ results = []
1801
+ try:
1802
+ lanes = survey(publish)
1803
+ if lanes is None:
1804
+ self._say_over_cap(container)
1805
+ return []
1806
+ for lane, pending, receipts in lanes:
1807
+ if len(results) >= MAX_PICKUPS:
1808
+ break
1809
+ if not pending:
1810
+ continue
1811
+ try:
1812
+ directory = os.open(lane, _CHILD_DIR, dir_fd=publish)
1813
+ except OSError:
1814
+ continue # swapped for a link or a file since the survey
1815
+ try:
1816
+ place = places.get(lane)
1817
+ ws = place[0] if place and place[1] == container else None
1818
+ results += self._lane(container, root, directory, lane, pending, receipts, ws, context,
1819
+ published, MAX_PICKUPS - len(results))
1820
+ finally:
1821
+ os.close(directory)
1822
+ finally:
1823
+ os.close(publish)
1824
+ if results:
1825
+ self.errors.pop(container, None)
1826
+ return results
1827
+
1828
+ def _say_over_cap(self, container):
1829
+ now = self.clock()
1830
+ if now - self.over_cap.get(container, float('-inf')) >= OVER_CAP_SAY_S:
1831
+ self.over_cap[container] = now
1832
+ print(f'lanes-docsd: pick-up {container}: refused: more than {MAX_ENTRIES} entries under '
1833
+ f'{L.CONTAINER_HOME}/.lz/state/publish; nothing is taken from this container until it holds fewer',
1834
+ file=sys.stderr, flush=True)
1835
+
1836
+ def _lane(self, container, root, directory, lane, pending, receipts, ws, context, published, budget):
1837
+ results, unclaimed = [], 0
1838
+ for request_id in pending:
1839
+ if len(results) >= budget or unclaimed >= MAX_PICKUPS:
1840
+ break
1841
+ outcome = self._take(container, root, directory, lane, request_id, ws, context, published)
1842
+ if outcome is None:
1843
+ break # the store is busy: the next cycle tries again
1844
+ if outcome is _UNCLAIMED:
1845
+ unclaimed += 1 # left for the next cycle; the next request is tried (F9)
1846
+ continue
1847
+ results.append((container, lane, request_id, outcome))
1848
+ receipts.append(request_id + RECEIPT_SUFFIX)
1849
+ if results:
1850
+ prune_receipts(directory, receipts)
1851
+ return results
1852
+
1853
+ def _allow(self, container, now):
1854
+ recent = [at for at in self.taken.get(container, ()) if now - at < PUBLISH_WINDOW]
1855
+ allowed = len(recent) < PUBLISHES
1856
+ self.taken[container] = recent + [now] if allowed else recent
1857
+ return allowed
1858
+
1859
+ def _claim(self, directory, request_id, claim, where):
1860
+ """Rename the request to its claim -> whether it was claimed. A directory planted at the claim's name is cleared
1861
+ (clear_planted) and the rename tried once more (F9); a failure is said once per lane (no file names)."""
1862
+ for attempt in (1, 2):
1863
+ try:
1864
+ os.rename(request_id + REQUEST_SUFFIX, claim, src_dir_fd=directory, dst_dir_fd=directory)
1865
+ return True
1866
+ except OSError as error:
1867
+ if attempt == 1 and error.errno in _PLANTED and clear_planted(directory, claim):
1868
+ continue
1869
+ self.error(where, _plain(error))
1870
+ return False
1871
+ return False
1872
+
1873
+ def _write_receipt(self, directory, name, data, where):
1874
+ """The receipt, by L.atomic_write_at (a rename onto `name`); a directory planted there is cleared first and the
1875
+ write tried once more (F9: before, the rename failed and the container never got its answer)."""
1876
+ for attempt in (1, 2):
1877
+ try:
1878
+ L.atomic_write_at(directory, name, data, mode=0o644)
1879
+ return True
1880
+ except OSError as error:
1881
+ if attempt == 1 and error.errno in _PLANTED and clear_planted(directory, name):
1882
+ continue
1883
+ self.error(where, _plain(error))
1884
+ except L.LanesError as error:
1885
+ self.error(where, error)
1886
+ return False
1887
+ return False
1888
+
1889
+ def _take(self, container, root, directory, lane, request_id, ws, context, published):
1890
+ """One request -> 'ok' or its refusal reason (docs.log's word); _UNCLAIMED when the claim could not be made
1891
+ (the request is left; the caller tries the next one); None when the store was busy (the claim is undone and
1892
+ the request waits for the next cycle)."""
1893
+ claim = f'.{request_id}{CLAIM_SUFFIX}'
1894
+ if not self._claim(directory, request_id, claim, f'{container}/{lane}'):
1895
+ return _UNCLAIMED
1896
+ now, login, meta = self.clock(), None, None
1897
+ try:
1898
+ request = read_request(directory, claim)
1899
+ if ws is None:
1900
+ raise Refused('unknown-lane')
1901
+ if not self._allow(container, now):
1902
+ raise Refused('rate-limited')
1903
+ if 'grants' not in context:
1904
+ context['grants'], context['profiles'] = member_grants(context['reg']), users.load_all()[0]
1905
+ login, fors = recipient(lane, ws, request['for'], context['grants'], context['profiles'])
1906
+ name, fd = open_published(root, request['path'])
1907
+ try:
1908
+ meta = D.intake(login, ws, lane, name, _Hashing(fd, request['claimed_sha256']), limit=PUBLISH_LIMIT,
1909
+ source='publish', kind=request['kind'], title=request['title'], for_logins=fors,
1910
+ by=lane)
1911
+ finally:
1912
+ os.close(fd)
1913
+ except D.Busy:
1914
+ try: # undo the claim: the request waits for the next cycle
1915
+ os.rename(claim, request_id + REQUEST_SUFFIX, src_dir_fd=directory, dst_dir_fd=directory)
1916
+ except OSError as error:
1917
+ self.error(f'{container}/{lane}', error)
1918
+ return None
1919
+ except Refused as error:
1920
+ refusal = error
1921
+ except D.TooLarge as error:
1922
+ refusal = Refused('too-large', size=error.received)
1923
+ except D.DocsError:
1924
+ refusal = Refused('invalid')
1925
+ except Exception as error: # noqa: BLE001 — claimed: the request is answered, never dropped silently
1926
+ self.error(f'{container}/{lane}', error)
1927
+ refusal = Refused('error')
1928
+ else:
1929
+ refusal = None
1930
+ if refusal is not None:
1931
+ receipt, refusal_word = receipt_of(request_id, now, refusal=refusal), refusal.reason
1932
+ else:
1933
+ receipt = receipt_of(request_id, now, meta=meta)
1934
+ published.append((lane, meta))
1935
+ outcome = 'ok' if meta is not None else refusal_word
1936
+ D.audit('publish', login, receipt['id'], lane, receipt['bytes'], receipt['sha256'], outcome, now)
1937
+ self._write_receipt(directory, request_id + RECEIPT_SUFFIX,
1938
+ (json.dumps(receipt, indent=1, ensure_ascii=False) + '\n').encode('utf-8'),
1939
+ f'{container}/{lane}')
1940
+ try:
1941
+ os.unlink(claim, dir_fd=directory)
1942
+ except IsADirectoryError:
1943
+ try:
1944
+ os.rmdir(claim, dir_fd=directory)
1945
+ except OSError:
1946
+ pass # a non-empty directory planted as a request: left under its dot name
1947
+ except OSError as error:
1948
+ self.error(f'{container}/{lane}', error)
1949
+ return outcome
1950
+
1951
+ def _push(self, lane, meta, names):
1952
+ now = self.clock()
1953
+ if now - self.pushed.get(lane, float('-inf')) < PUBLISHED_PUSH_S:
1954
+ return None
1955
+ self.pushed[lane] = now
1956
+ try:
1957
+ return N.notify_published(lane, meta, names, conf_path=self.notify_conf, state=L.STATE, now=now,
1958
+ opener=self.opener)
1959
+ except Exception as error: # noqa: BLE001 — a push is best effort
1960
+ self.error('notify', error)
1961
+ return None
1962
+
1963
+
1964
+ def pin_state():
1965
+ mode, problem = E.read_mode()
1966
+ if mode == 'request':
1967
+ return f'pinned request ({E.mode_path()})'
1968
+ return (f'NOT pinned ({problem or ("pinned " + mode if mode else "auto")}): every request is refused until the '
1969
+ 'Serve header check passes and `lanes enrol mode request` pins it')
1970
+
1971
+
1972
+ def main(argv=None):
1973
+ parser = argparse.ArgumentParser(prog='lanes-docsd', description='Lanes docs service (loopback only).')
1974
+ parser.add_argument('--port', type=int, default=None, help=f'default $LANES_DOCS_PORT or {DEFAULT_PORT}')
1975
+ parser.add_argument('--bind', default=DEFAULT_BIND, help=f'a loopback address (default {DEFAULT_BIND})')
1976
+ args = parser.parse_args(argv)
1977
+ port = args.port
1978
+ if port is None:
1979
+ try:
1980
+ port = int(os.environ.get('LANES_DOCS_PORT') or DEFAULT_PORT)
1981
+ except ValueError:
1982
+ parser.error('LANES_DOCS_PORT is not a port number')
1983
+ if not 0 <= port <= 65535:
1984
+ parser.error(f'{port} is not a port number')
1985
+ try:
1986
+ loopback = ipaddress.ip_address(args.bind).is_loopback
1987
+ except ValueError:
1988
+ loopback = False
1989
+ if not loopback:
1990
+ parser.error('--bind takes a loopback address: the service trusts the identity headers of its proxy chain')
1991
+ service = Service.from_environ(os.environ)
1992
+ server = Server((args.bind, port), service)
1993
+ L.record_tree('docsd') # W9 U0: the tree this service runs from (`lanes version`, doctor)
1994
+ # The publish pick-up (plan W10 W4): its own 5 s cycle on a daemon thread, beside the HTTP server; each cycle also
1995
+ # writes the service's due `denied-repeat` lines (F4).
1996
+ threading.Thread(target=Pickup(tick=service.flush_denied).run, name='lanes-docsd-pickup', daemon=True).start()
1997
+ print(f'lanes-docsd {version()}: listening on {args.bind}:{server.server_address[1]}; {pin_state()}; '
1998
+ f'domains {",".join(service.domains)}; store {D.root()}; publish pick-up every {PICKUP_INTERVAL} s',
1999
+ file=sys.stderr, flush=True)
2000
+ try:
2001
+ server.serve_forever()
2002
+ except KeyboardInterrupt:
2003
+ pass
2004
+ finally:
2005
+ server.server_close()
2006
+ return 0
2007
+
2008
+
2009
+ if __name__ == '__main__':
2010
+ sys.exit(main())