@vibelord/lanes 1.2.0 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/payload/lanes/README.md +4 -3
- package/payload/lanes/VERSION +1 -1
- package/payload/lanes/bin/lanes +301 -45
- package/payload/lanes/container/lane +42 -1
- package/payload/lanes/container/lane-mcp +65 -6
- package/payload/lanes/container/lane_core.py +178 -0
- package/payload/lanes/docs/ARCHITECTURE.md +192 -20
- package/payload/lanes/docs/DESIGN.md +6 -3
- package/payload/lanes/docs/INSTALL.md +25 -8
- package/payload/lanes/docs/RUNBOOK.md +369 -8
- package/payload/lanes/docs/USERGUIDE.md +201 -25
- package/payload/lanes/holders.lock +5 -0
- package/payload/lanes/install/enable-docs.sh +78 -0
- package/payload/lanes/install/host-install.sh +39 -5
- package/payload/lanes/libexec/docs_store.py +995 -0
- package/payload/lanes/libexec/docsd.py +2010 -0
- package/payload/lanes/libexec/mdlite.py +575 -0
- package/payload/lanes/libexec/members.py +5 -0
- package/payload/lanes/libexec/notify.py +65 -5
- package/payload/lanes/libexec/sidebar.py +106 -13
- package/payload/lanes/libexec/statusd.py +97 -5
- package/payload/lanes/libexec/tokens.py +31 -9
- package/payload/lanes/libexec/update.py +94 -22
- package/payload/lanes/libexec/uploads.py +538 -0
- package/payload/lanes/systemd/lanes-docsd.service +29 -0
- package/payload/lanes/web/Caddyfile +24 -1
- package/payload/lanes/web/README.md +86 -8
- package/payload/lanes/web/browser.js +2 -4
- package/payload/lanes/web/docs.css +52 -0
- package/payload/lanes/web/docs.js +36 -0
- package/payload/lanes/web/mobile.js +3 -2
- package/payload/lanes/web/upload.css +17 -0
- package/payload/lanes/web/upload.js +84 -0
- package/payload/manifest.json +4 -3
|
@@ -0,0 +1,2010 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""lanes-docsd — the docs cockpit's web service (W10 W3, LNS-104).
|
|
3
|
+
|
|
4
|
+
What an agent publishes from a lane, and what a person uploads, is opened here from the phone or laptop shell over the
|
|
5
|
+
tailnet, bound to the person's login. `lanes-docsd.service` runs this file on 127.0.0.1:8185 (`LANES_DOCS_PORT`;
|
|
6
|
+
`--port`, `--bind`: a loopback address only — the service trusts the identity headers its proxy chain sets), behind the
|
|
7
|
+
loopback Caddy of `lanes-mobile.service` (`handle_path /lanes-docs/*`, which strips `/lanes-docs`) behind Tailscale
|
|
8
|
+
Serve (`https://<host>.<tailnet>.ts.net:8443/lanes-docs/...`). Standard library only. The documents, their receipts,
|
|
9
|
+
counts and docs.log are docs_store.py's (W1); Markdown is rendered by mdlite.py (W2).
|
|
10
|
+
|
|
11
|
+
Who is served (plan "Auth and threat model", binding). A request is answered only when ALL hold, checked in this order
|
|
12
|
+
by Service.gate, each refusal a 403 with its word:
|
|
13
|
+
not-through-serve enrold.through_serve: the X-Forwarded-For Serve sets and Caddy keeps names a non-loopback
|
|
14
|
+
address (a probe of 127.0.0.1:8185, or of Caddy from this host, carries none)
|
|
15
|
+
no-identity enrold.caller_of(headers).present: Serve's Tailscale-User-Login and -Name header set (a tagged
|
|
16
|
+
device has none)
|
|
17
|
+
invalid-login the login validates as members.py stores one
|
|
18
|
+
not-pinned enrold.read_mode()[0] == 'request': the operator ran the RUNBOOK's Serve header check and pinned
|
|
19
|
+
`lanes enrol mode request`. Unpinned (auto), pinned `invite` or unreadable: refused — docsd never
|
|
20
|
+
latches anything from traffic and never reads LANES_ENROL_REQUEST_MODE
|
|
21
|
+
not-a-member login.casefold() in enrold.member_logins(): the login owns a valid members/<device>.json
|
|
22
|
+
domain-not-allowed enrold's domain rule over LANES_DOCS_DOMAINS (else LANES_ENROL_DOMAINS; default beyondwork.ai;
|
|
23
|
+
comma/space separated, `*` = any)
|
|
24
|
+
cross-site fetch_site_ok, on EVERY route (security review 2026-10-08, F2): the request's Sec-Fetch-Site
|
|
25
|
+
is `same-origin` or `none` (a page, script, stylesheet or image of this site, or a navigation the
|
|
26
|
+
person started: the address bar, a bookmark, a link opened from a terminal or another app) — or
|
|
27
|
+
it is a top-level navigation, a GET or HEAD with Sec-Fetch-Mode `navigate` and Sec-Fetch-Dest
|
|
28
|
+
`document`, whatever its Sec-Fetch-Site (a link tapped in ntfy's web app, a chat or a mail).
|
|
29
|
+
Everything else from another site is refused: a sandboxed artifact is an opaque origin, so its
|
|
30
|
+
fetch() and subresource loads are `cross-site` (mode cors/no-cors, dest empty/image/script/…),
|
|
31
|
+
and *.ts.net is a public suffix, so another tailnet host's page is `same-site`; neither may spend
|
|
32
|
+
the viewer's login here. POST /up keeps the stricter rule: `same-origin` or `none` only, a
|
|
33
|
+
navigation included (a form posted from another site). An absent Sec-Fetch-Site is refused
|
|
34
|
+
unless the request is such a navigation: every browser that reaches Serve sends the headers
|
|
35
|
+
(Chrome 76, Firefox 90, Safari 16.4). The artifact's sandbox (`sandbox allow-scripts`, nothing
|
|
36
|
+
else: no allow-popups, allow-top-navigation, allow-forms or allow-downloads) cannot open a
|
|
37
|
+
window, submit a form or navigate another browsing context; the one navigation its script can
|
|
38
|
+
start is of its own page, which replaces it (its script ends) unless the answer is a download the
|
|
39
|
+
sandbox then blocks — so what it can request through the navigation exemption is bounded by the
|
|
40
|
+
per-login limit below (VIEWS per VIEW_WINDOW; the HTML standard's rules, not measured in a
|
|
41
|
+
browser).
|
|
42
|
+
No Zellij cookie is consulted, ever. DEF-1645 (a process that reaches loopback can send the headers itself) stays the
|
|
43
|
+
documented open risk; the pin gate and the Serve-only check are the controls. Never put this behind Tailscale Funnel.
|
|
44
|
+
A refusal is one docs.log `denied` line per source (the tailnet address Serve forwarded, else the peer) and word; the
|
|
45
|
+
same source refused again within DENIED_REPEAT_S adds no line, and one `denied-repeat` line per DENIED_REPEAT_S says how
|
|
46
|
+
many it folded (its `bytes` field is that count); the login is logged at most MAX_DENIED_LOGIN characters (F4).
|
|
47
|
+
|
|
48
|
+
Limits (security review 2026-10-08, F2/F5/F8; per process): an authenticated login makes at most VIEWS requests other
|
|
49
|
+
than POST /up in VIEW_WINDOW s (more: 429 rate-limited + Retry-After, one docs.log line per window); at most
|
|
50
|
+
RENDER_SLOTS Markdown renders run at once (a view waits RENDER_WAIT_S for one, then 503 busy); a rendered Markdown
|
|
51
|
+
document is kept in memory by its sha256, least recently used out first, at most RENDER_CACHE_BYTES in all and
|
|
52
|
+
RENDER_CACHE_ENTRY each, so viewing it again renders nothing; UPLOAD_SLOTS big buffers at once, shared by uploads and
|
|
53
|
+
decryptions, and at most one upload per login; an upload's body must arrive within BODY_GRACE_S + its length /
|
|
54
|
+
BODY_MIN_RATE seconds, else 408 request-timeout and its slot is free again.
|
|
55
|
+
|
|
56
|
+
Who sees a document (visible). docs_store's own rule lets a login see what its store holds, what `for` names it in,
|
|
57
|
+
and every document whose `for` is empty; docsd narrows the last case. A document is visible to the caller when
|
|
58
|
+
* it is in the caller's own store (docs/<stem(caller)>/…: an upload of theirs), or
|
|
59
|
+
* its `for` is not empty and names the caller (casefolded), or
|
|
60
|
+
* its `for` is empty and a member record the caller owns grants the document's workspace (member_workspaces): a
|
|
61
|
+
record's `workspaces` (the board ids an invite recorded with `lanes invite --workspaces`) grants those boards, and
|
|
62
|
+
a record that lists none (empty or absent: a key added by hand, a request, an invite minted without
|
|
63
|
+
`--workspaces`) grants every workspace this host serves (the registry's). Nothing else in Lanes narrows by that
|
|
64
|
+
list — a view shows every board — so an empty list never meant "no board" (DEF-1852). The upload form's lanes
|
|
65
|
+
(upload_targets) and `POST /up` follow the same grant.
|
|
66
|
+
A document the caller cannot see is answered 404, exactly like one that does not exist.
|
|
67
|
+
|
|
68
|
+
Publish pick-up (plan W10 W4; Pickup, a daemon thread main() starts, on its own PICKUP_INTERVAL = 5 s cycle — never
|
|
69
|
+
statusd's). `lane publish <file>` and lane-mcp's `lanes_publish` write ~/.lz/state/publish/<lane>/<id>.json in a
|
|
70
|
+
workspace container (DM-LANES-PUBLISH-REQUEST, lane_core.publish). Each cycle, for every container containers.json
|
|
71
|
+
names whose init pid is alive, whose /proc/<pid>/root is not the host's own `/` (a recycled pid), and — when this
|
|
72
|
+
account can see the home volume containers.json records for it — whose /home/coder is that volume (same st_dev and
|
|
73
|
+
st_ino: a pid the kernel gave another container is skipped; F10), the pick-up first surveys the container's publish
|
|
74
|
+
directory and its lane directories (os.scandir, opened without following a link, `L.open_real_dir`): at most
|
|
75
|
+
MAX_ENTRIES entries in all are examined per container per cycle, nothing more is read, and a container that holds more
|
|
76
|
+
is refused for the cycle — nothing is taken from it, and the journal says so once per OVER_CAP_SAY_S (F1: listing
|
|
77
|
+
100,000 planted lane directories took 5.7 s and 116 MiB a cycle; ~650,000 crossed MemoryMax). Then, lane by lane, at
|
|
78
|
+
most MAX_PICKUPS requests per container:
|
|
79
|
+
1. claim: `<id>.json` is renamed to `.<id>.taking` in the same directory (nothing is read from a directory docsd
|
|
80
|
+
cannot write: a request it could not remove would be taken again every cycle). A directory planted at the claim's
|
|
81
|
+
name is removed when empty, else moved aside to a dot name; a request that still cannot be claimed is skipped and
|
|
82
|
+
the next one is taken (F9: one planted `.taking` directory stalled its lane's queue);
|
|
83
|
+
2. read it as a regular file of at most MAX_REQUEST bytes (`L.read_regular_at`: never a link or a FIFO) and check
|
|
84
|
+
it: `v` 1, `path` absolute, `kind` md|html|svg|file, `claimed_sha256` 64 hex, `bytes` <= PUBLISH_LIMIT (the
|
|
85
|
+
docs limit, 32 MiB), `title`/`for`/`mime` strings or null; the lane must be one the registry places in THIS
|
|
86
|
+
container; PUBLISHES requests per container per PUBLISH_WINDOW s;
|
|
87
|
+
3. resolve `path` inside the container (`L.container_realpath`: a link restarts at the container's root, never the
|
|
88
|
+
host's) to a file under /home/coder, open every directory without following a link (`L.open_real_dir`) and the
|
|
89
|
+
file O_NOFOLLOW|O_NONBLOCK, a regular file of at most PUBLISH_LIMIT bytes: a link swapped in meanwhile, a FIFO,
|
|
90
|
+
a device or a larger file is refused, so a container never makes the host read outside its home;
|
|
91
|
+
4. stream it into docs_store.intake(login, ws, lane, name, …, limit=PUBLISH_LIMIT, source='publish', kind, title,
|
|
92
|
+
for_logins, by=<lane id>) through a reader that hashes as it goes and, at the end, refuses a sha256 other than
|
|
93
|
+
the claimed one inside intake (which then removes what it wrote: nothing is stored);
|
|
94
|
+
5. write `<id>.receipt.json` back beside the request (`L.atomic_write_at`, 0644; the statusd write_container_status
|
|
95
|
+
pattern: replaced by rename, never written through a link; a directory planted at that name is removed or moved
|
|
96
|
+
aside first, F9; DM-LANES-PUBLISH-RECEIPT) — {v: 1, id, request, sha256, bytes, url, at, outcome: ok} or {…, id:
|
|
97
|
+
null, url: null, outcome: refused, reason} — remove the claim, keep the newest MAX_RECEIPTS receipts of the lane,
|
|
98
|
+
and append one docs.log `publish` line (outcome `ok` or the reason). The counts follow through intake. A busy
|
|
99
|
+
store leaves the claim's request for the next cycle.
|
|
100
|
+
Refusal reasons (docs.log): unreadable-request, bad-version, bad-request, bad-kind, bad-title, too-large, unknown-lane,
|
|
101
|
+
rate-limited, for-not-a-member, no-recipient, outside-home, not-a-file, sha256-mismatch, invalid (intake refused),
|
|
102
|
+
error (anything else after the claim: the journal has it). The receipt says the same word, except that
|
|
103
|
+
for-not-a-member and no-recipient both say `not-delivered` (F6: the two words told a container whether a login it
|
|
104
|
+
named owns a member record; a `for` naming a member still answers ok). Without a registry.json nothing is taken (a
|
|
105
|
+
request waits).
|
|
106
|
+
|
|
107
|
+
Who a published document is for (recipient): the request's `for` — a login that owns a member record (else refused
|
|
108
|
+
`for-not-a-member`), stored in that login's store with `for` = [it]; else the member whose profile's `last_opened.lane`
|
|
109
|
+
is this lane, the latest `at` winning (users/<login>.json), the same way; else every member whose record grants the
|
|
110
|
+
lane's workspace (member_workspaces' rule): `for` empty, stored in the store of the first of them by casefolded login
|
|
111
|
+
(none: refused `no-recipient`). Logins are spelled as their member record spells them. A successful pick-up sends one
|
|
112
|
+
`published` push through notify (notify.notify_published: notify.conf, the desk-presence rule, at most one per lane
|
|
113
|
+
per PUBLISHED_PUSH_S), best effort, after the cycle's receipts are written.
|
|
114
|
+
|
|
115
|
+
Wire contract v1 (DM-LANES-DOCS-WIRE; paths as docsd sees them, after Caddy strips /lanes-docs; every link and asset
|
|
116
|
+
reference a page carries is RELATIVE, so the pages work under that prefix):
|
|
117
|
+
GET / 200 page: the caller's visible documents, workspace -> lane -> newest first, with counts;
|
|
118
|
+
the upload form (the lanes of the caller's workspaces); an empty state
|
|
119
|
+
GET /?ws=<ws>&lane=<lane> the same, only that workspace and/or lane (the sidebar's and the shells' deep link for a
|
|
120
|
+
lane is exactly `/lanes-docs/?ws=<ws>&lane=<lane>`, W7). Either may be left out. A value
|
|
121
|
+
must be known to the caller (index_filter: a workspace a member record of theirs lists,
|
|
122
|
+
or one a visible document is in; a lane of the registry in such a workspace, or one a
|
|
123
|
+
visible document is in, and in the named workspace when both are given) — anything
|
|
124
|
+
else, a repeated or empty value included, is 404 not-found
|
|
125
|
+
GET /d/<id> 200 page by kind: md -> mdlite.render() in the page (503 busy + Retry-After when no
|
|
126
|
+
render slot frees within RENDER_WAIT_S; a document rendered before comes from the cache);
|
|
127
|
+
html|svg -> title, facts and a link to d/<id>/artifact (never embedded, no iframe);
|
|
128
|
+
upload|file -> facts and a download link
|
|
129
|
+
GET /d/<id>/raw 200 the stored bytes: Content-Disposition: attachment; filename="<name>", X-Lanes-Sha256
|
|
130
|
+
GET /d/<id>/artifact 200 for kind html|svg only (else 404): the bytes as a top-level document with
|
|
131
|
+
Content-Security-Policy: sandbox allow-scripts (nothing else) — an opaque origin
|
|
132
|
+
An ENCRYPTED document (docs_store's `age` switch, meta `enc`): raw and artifact decrypt
|
|
133
|
+
it on the fly when LANES_DOCS_AGE_IDENTITY names an identity file (at most MAX_BODY
|
|
134
|
+
bytes, one at a time, checked against the receipt), else 409 encrypted (a page: fetch
|
|
135
|
+
the copy in the workspace's uploads/); its view shows the facts, Markdown rendered only
|
|
136
|
+
when it can be decrypted
|
|
137
|
+
GET /receipt/<id>.json 200 the document's receipt.json bytes (DM-LANES-DOCS-RECEIPT), application/json
|
|
138
|
+
GET /docs.css, /docs.js 200 lanes/web/docs.css|docs.js of the tree this process runs (W6 writes them), else 404
|
|
139
|
+
GET /upload.css, /upload.js 200 lanes/web/upload.css|upload.js (W5: the index's drop area), else 404
|
|
140
|
+
POST /up multipart/form-data: `lane` (a lane id of the registry), optional `ws` (must be that
|
|
141
|
+
lane's workspace), one or more file parts (a part with a filename). The workspace must be
|
|
142
|
+
one the caller's member records grant. Each file goes through uploads.receive (W5): it
|
|
143
|
+
lands in the workspace's uploads/ folder in its container (`<folder>/uploads/<name>`,
|
|
144
|
+
name docs_store.clean_name's, `-1` on a collision, exactly as `lanes upload` lands one)
|
|
145
|
+
AND docs_store.intake(login, ws, lane, name, …, source='web', kind by extension (md,
|
|
146
|
+
html, svg, else file), for=[caller], by=caller) in one pass, both or neither, and is
|
|
147
|
+
answered with a receipt {id, name, sha256, bytes, url, at, landed} (`landed` the
|
|
148
|
+
in-container path an agent reads): 200 JSON {receipts} (an
|
|
149
|
+
Accept without text/html) or a page. Refused: 429 rate-limited + Retry-After (UPLOADS
|
|
150
|
+
per login in UPLOAD_WINDOW s, counted when the attempt starts), 503 busy + Retry-After
|
|
151
|
+
(UPLOAD_SLOTS uploads and decryptions at once, one per login, or the store lock), 415
|
|
152
|
+
not multipart/form-data, 411 no Content-Length (or a chunked body), 413 over MAX_BODY (32
|
|
153
|
+
MiB), 408 request-timeout (the body did not arrive within BODY_GRACE_S + length /
|
|
154
|
+
BODY_MIN_RATE s), 400 bad-request (a
|
|
155
|
+
malformed form) | no-file | no-lane | unknown-lane | unknown-workspace, 503
|
|
156
|
+
registry-unavailable (no registry.json to resolve the lane in), 503 uploads-unavailable
|
|
157
|
+
(the workspace's container folder cannot be written: not running, a link on the way),
|
|
158
|
+
500 encrypt-failed (the `age` switch is on and age failed). The gate's `cross-site`
|
|
159
|
+
(403) applies to an upload with no navigation exemption: same-origin or none only
|
|
160
|
+
every route but POST /up 429 rate-limited + Retry-After past VIEWS requests per login in VIEW_WINDOW s
|
|
161
|
+
everything else 404 not-found (an id is matched against docs_store's id rule before any path is formed;
|
|
162
|
+
`..`, encoded dots and overlong ids never match); a known path with another method 405
|
|
163
|
+
Errors are JSON {"error": <word>}, or a short page when the Accept header names text/html. A request body left unread
|
|
164
|
+
by a refusal is read away up to its Content-Length (at most DRAIN_LIMIT) BEFORE the answer, and the answer carries
|
|
165
|
+
`Connection: close`; past DRAIN_LIMIT the answer is sent, the write side shut and the rest read away for LINGER_S
|
|
166
|
+
seconds, so the client still reads the refusal instead of a reset (DEF-1854).
|
|
167
|
+
|
|
168
|
+
Headers. Every response: Cache-Control no-store (static files no-cache), X-Content-Type-Options nosniff,
|
|
169
|
+
X-Frame-Options DENY, Referrer-Policy no-referrer and a Content-Security-Policy: a page PAGE_POLICY, an artifact
|
|
170
|
+
exactly ARTIFACT_POLICY, everything else DATA_POLICY. The site-wide Caddy header block skips /lanes-docs/* (`header
|
|
171
|
+
@notdocs`), so these are the only ones a document gets.
|
|
172
|
+
|
|
173
|
+
docs.log (DM-LANES-DOCS-LOG, through docs_store.audit; never content, a title or a name): `denied` (a gate refusal,
|
|
174
|
+
the outcome is its word; coalesced per source, with `denied-repeat` lines, see the gate above), `view` `raw` `artifact`
|
|
175
|
+
`receipt` (ok | not-found | encrypted | busy | rate-limited), `upload` (a refusal before any file reaches intake;
|
|
176
|
+
intake writes its own `intake` line per file) and `publish` (one per request the pick-up takes: the store login, the
|
|
177
|
+
document id, the lane, the host's bytes and sha256 when known, `ok` or the refusal reason).
|
|
178
|
+
"""
|
|
179
|
+
import argparse
|
|
180
|
+
import collections
|
|
181
|
+
import email.message
|
|
182
|
+
import email.parser
|
|
183
|
+
import email.policy
|
|
184
|
+
import errno
|
|
185
|
+
import hashlib
|
|
186
|
+
import html
|
|
187
|
+
import ipaddress
|
|
188
|
+
import json
|
|
189
|
+
import os
|
|
190
|
+
import re
|
|
191
|
+
import socket
|
|
192
|
+
import socketserver
|
|
193
|
+
import stat
|
|
194
|
+
import sys
|
|
195
|
+
import threading
|
|
196
|
+
import time
|
|
197
|
+
import traceback
|
|
198
|
+
import urllib.parse
|
|
199
|
+
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
200
|
+
from pathlib import Path
|
|
201
|
+
|
|
202
|
+
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
203
|
+
import lanes_lib as L # noqa: E402
|
|
204
|
+
import docs_store as D # noqa: E402
|
|
205
|
+
import enrold as E # noqa: E402
|
|
206
|
+
import keys as K # noqa: E402
|
|
207
|
+
import mdlite # noqa: E402
|
|
208
|
+
import notify as N # noqa: E402
|
|
209
|
+
import uploads as U # noqa: E402
|
|
210
|
+
import users # noqa: E402
|
|
211
|
+
|
|
212
|
+
WIRE_VERSION = 1
|
|
213
|
+
DEFAULT_PORT = 8185 # web/Caddyfile `handle_path /lanes-docs/*` forwards here
|
|
214
|
+
DEFAULT_BIND = '127.0.0.1'
|
|
215
|
+
MAX_BODY = 32 * 1024 * 1024 # a POST /up body; also each file's intake limit (uploads.UPLOAD_LIMIT)
|
|
216
|
+
DRAIN_LIMIT = 2 * MAX_BODY # an unread body is read away up to this BEFORE a refusal is sent (DEF-1854)
|
|
217
|
+
LINGER_S = 5 # past DRAIN_LIMIT: seconds the rest is read away after the answer
|
|
218
|
+
MAX_PARTS = 64 # parts of one form (fields and files)
|
|
219
|
+
MAX_PART_HEADER = 8 * 1024 # one part's header block
|
|
220
|
+
MAX_FIELD = 1024 # one non-file field's value
|
|
221
|
+
MAX_CONTENT_TYPE = 512
|
|
222
|
+
MAX_STATIC = 512 * 1024 # docs.css / docs.js
|
|
223
|
+
UPLOADS = 30 # POST /up attempts per login in UPLOAD_WINDOW seconds
|
|
224
|
+
UPLOAD_WINDOW = 600
|
|
225
|
+
UPLOAD_SLOTS = 2 # big buffers at once (<= MAX_BODY each): uploads parsed and documents decrypted
|
|
226
|
+
BUSY_RETRY = 5 # Retry-After of a 503 busy
|
|
227
|
+
DECRYPT_WAIT_S = BUSY_RETRY # a decryption waits this long for its turn and a slot, then 503 busy
|
|
228
|
+
MAX_LOGINS = 4096 # logins a limiter remembers
|
|
229
|
+
BODY_GRACE_S = 30 # an upload body must arrive within BODY_GRACE_S + its length / BODY_MIN_RATE s
|
|
230
|
+
BODY_MIN_RATE = 64 * 1024 # (bytes per second: 32 MiB in 542 s), else 408 and the slot is free (F5)
|
|
231
|
+
VIEWS = 300 # requests other than POST /up per login in VIEW_WINDOW seconds (F2)
|
|
232
|
+
VIEW_WINDOW = 60
|
|
233
|
+
RENDER_SLOTS = 2 # Markdown renders at once: a hostile 1 MiB document renders for seconds (F2)
|
|
234
|
+
RENDER_WAIT_S = 1.0 # a view waits this long for a render slot, then 503 busy
|
|
235
|
+
RENDER_CACHE_BYTES = 8 * 1024 * 1024 # rendered Markdown kept in memory by the document's sha256, least recent out
|
|
236
|
+
RENDER_CACHE_ENTRY = 2 * 1024 * 1024 # a larger render is not kept
|
|
237
|
+
DENIED_REPEAT_S = 60 # one docs.log line per refused source and word in this many seconds (F4)
|
|
238
|
+
MAX_DENIED_SOURCES = 1024 # sources the refusal coalescing remembers
|
|
239
|
+
MAX_DENIED_LOGIN = 64 # characters of an unauthenticated login a `denied` line keeps
|
|
240
|
+
CHUNK = 64 * 1024
|
|
241
|
+
PAGE_POLICY = ("default-src 'none'; style-src 'self' 'unsafe-inline'; script-src 'self'; img-src 'self' data:; "
|
|
242
|
+
"connect-src 'self'; form-action 'self'; base-uri 'none'") # connect-src: upload.js's POST /up
|
|
243
|
+
ARTIFACT_POLICY = 'sandbox allow-scripts'
|
|
244
|
+
DATA_POLICY = "default-src 'none'; sandbox"
|
|
245
|
+
SHA_HEADER = 'X-Lanes-Sha256'
|
|
246
|
+
ARTIFACT_TYPES = {'html': 'text/html; charset=utf-8', 'svg': 'image/svg+xml'}
|
|
247
|
+
STATIC = {'docs.css': 'text/css; charset=utf-8', 'docs.js': 'text/javascript; charset=utf-8',
|
|
248
|
+
'upload.css': 'text/css; charset=utf-8', 'upload.js': 'text/javascript; charset=utf-8'}
|
|
249
|
+
FETCH_SITES = ('same-origin', 'none') # Sec-Fetch-Site values the gate serves (every route, F2) …
|
|
250
|
+
NAVIGATION_METHODS = ('GET', 'HEAD') # … and a top-level navigation by these methods, from any site
|
|
251
|
+
_DECRYPTS = threading.BoundedSemaphore(1) # encrypted documents decrypted at once (each held in memory)
|
|
252
|
+
_ID = r'([a-z0-9]{10,20})' # docs_store.ID_RE; check_id runs again before any path is formed
|
|
253
|
+
ROUTES = (('index', re.compile(r'/', re.ASCII)),
|
|
254
|
+
('view', re.compile(rf'/d/{_ID}', re.ASCII)),
|
|
255
|
+
('raw', re.compile(rf'/d/{_ID}/raw', re.ASCII)),
|
|
256
|
+
('artifact', re.compile(rf'/d/{_ID}/artifact', re.ASCII)),
|
|
257
|
+
('receipt', re.compile(rf'/receipt/{_ID}\.json', re.ASCII)),
|
|
258
|
+
('up', re.compile(r'/up', re.ASCII)),
|
|
259
|
+
('static', re.compile(r'/(docs\.css|docs\.js|upload\.css|upload\.js)', re.ASCII)))
|
|
260
|
+
METHODS = {'up': ('POST',)} # every other route: GET
|
|
261
|
+
_MIME = re.compile(r'[a-z0-9][a-z0-9.+-]{0,63}/[a-z0-9][a-z0-9.+-]{0,63}', re.ASCII)
|
|
262
|
+
_FILENAME_UNSAFE = re.compile(r'[^A-Za-z0-9._-]')
|
|
263
|
+
_BOUNDARY = re.compile(r"[0-9A-Za-z'()+_,./:=? -]{0,69}[0-9A-Za-z'()+_,./:=?-]", re.ASCII) # RFC 2046 bchars
|
|
264
|
+
_CTE_OK = ('7bit', '8bit', 'binary')
|
|
265
|
+
_CONTROL = re.compile(r'[\x00-\x1f\x7f]')
|
|
266
|
+
|
|
267
|
+
SENTENCES = {
|
|
268
|
+
'not-through-serve': 'This page answers only requests that come through Tailscale Serve, at the server\'s '
|
|
269
|
+
'tailnet HTTPS address.',
|
|
270
|
+
'no-identity': 'No tailnet identity reached this page: open it from a device signed in to the tailnet as yourself '
|
|
271
|
+
'(a tagged device has no login).',
|
|
272
|
+
'invalid-login': 'The tailnet login this request carries is not one Lanes accepts.',
|
|
273
|
+
'not-pinned': 'The docs service is off until the operator has run the Serve header check and pinned it '
|
|
274
|
+
'(`lanes enrol mode request` on the server).',
|
|
275
|
+
'not-a-member': 'Your tailnet login owns no member record on this server.',
|
|
276
|
+
'domain-not-allowed': 'Your tailnet login is not in a domain this server accepts.',
|
|
277
|
+
'not-found': 'No such document here, or it is not one you can open.',
|
|
278
|
+
'method-not-allowed': 'That method is not used here.',
|
|
279
|
+
'cross-site': 'Refused: this request came from another site and was not a link you followed (Sec-Fetch-Site), '
|
|
280
|
+
'or your browser does not say where it came from. Open the Docs page from the shell or the sidebar '
|
|
281
|
+
'link, or type its address.',
|
|
282
|
+
'rate-limited': 'Too many requests in a short time; try again in a few minutes.',
|
|
283
|
+
'busy': 'The docs service is busy; try again in a moment.',
|
|
284
|
+
'request-timeout': 'The upload arrived too slowly and was stopped; nothing was stored. Try again.',
|
|
285
|
+
'unsupported-media-type': 'An upload is a multipart/form-data form.',
|
|
286
|
+
'length-required': 'An upload needs a Content-Length.',
|
|
287
|
+
'too-large': 'The upload is larger than 32 MiB.',
|
|
288
|
+
'bad-request': 'That upload form could not be read.',
|
|
289
|
+
'no-file': 'The form carried no file.',
|
|
290
|
+
'no-lane': 'Choose the lane the file is for.',
|
|
291
|
+
'unknown-lane': 'That lane is not one of this server\'s lanes.',
|
|
292
|
+
'unknown-workspace': 'That workspace is not one your member record lists.',
|
|
293
|
+
'registry-unavailable': 'The server has no lane registry yet (`lanes sync`).',
|
|
294
|
+
'uploads-unavailable': 'The workspace\'s uploads/ folder cannot be written now (its container is not running, or '
|
|
295
|
+
'the folder is reached through a link); nothing was stored.',
|
|
296
|
+
'encrypt-failed': 'The docs store encrypts at rest and age failed; the journal says why. Nothing was stored.',
|
|
297
|
+
'encrypted': 'This document is encrypted at rest and the docs service holds no key to open it: fetch the copy in '
|
|
298
|
+
'the workspace\'s uploads/ folder (or the lane\'s own file) instead.',
|
|
299
|
+
'server-error': 'Something went wrong on the server; the journal says what.',
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
|
|
303
|
+
# ------------------------------------------------------------------------------------------------------- basics
|
|
304
|
+
|
|
305
|
+
def version():
|
|
306
|
+
return E.version()
|
|
307
|
+
|
|
308
|
+
|
|
309
|
+
def web_dir():
|
|
310
|
+
"""lanes/web of the tree this process runs (L.REPO: the unit names the `current` link, so the installed release),
|
|
311
|
+
where docs.css and docs.js live (W6)."""
|
|
312
|
+
return L.REPO / 'web'
|
|
313
|
+
|
|
314
|
+
|
|
315
|
+
def esc(value):
|
|
316
|
+
return html.escape('' if value is None else str(value), quote=True)
|
|
317
|
+
|
|
318
|
+
|
|
319
|
+
def human_bytes(size):
|
|
320
|
+
if size < 1024:
|
|
321
|
+
return f'{size} B'
|
|
322
|
+
if size < 1024 * 1024:
|
|
323
|
+
return f'{size / 1024:.1f} KiB'
|
|
324
|
+
return f'{size / (1024 * 1024):.1f} MiB'
|
|
325
|
+
|
|
326
|
+
|
|
327
|
+
def wants_html(headers):
|
|
328
|
+
return 'text/html' in (headers.get('Accept') or '').lower()
|
|
329
|
+
|
|
330
|
+
|
|
331
|
+
def download_name(name):
|
|
332
|
+
"""The filename a Content-Disposition carries: docs_store names are already [A-Za-z0-9._-]; anything else is `_`."""
|
|
333
|
+
cleaned = _FILENAME_UNSAFE.sub('_', name or '')[:D.MAX_NAME + D.NAME_SLACK]
|
|
334
|
+
return cleaned or 'file'
|
|
335
|
+
|
|
336
|
+
|
|
337
|
+
def content_type_of(meta):
|
|
338
|
+
mime = meta.get('mime')
|
|
339
|
+
return mime if isinstance(mime, str) and _MIME.fullmatch(mime) else 'application/octet-stream'
|
|
340
|
+
|
|
341
|
+
|
|
342
|
+
# ------------------------------------------------------------------------------------------------------- people
|
|
343
|
+
|
|
344
|
+
def served_workspaces(reg):
|
|
345
|
+
"""The workspace ids the registry `reg` serves -> a set (empty without a registry)."""
|
|
346
|
+
return {workspace['id'] for workspace in (reg or {}).get('workspaces') or []
|
|
347
|
+
if isinstance(workspace, dict) and isinstance(workspace.get('id'), str)}
|
|
348
|
+
|
|
349
|
+
|
|
350
|
+
def _listed(member):
|
|
351
|
+
return {ws for ws in member.get('workspaces') or () if isinstance(ws, str)}
|
|
352
|
+
|
|
353
|
+
|
|
354
|
+
def member_workspaces(login, reg=None):
|
|
355
|
+
"""The workspaces the member records `login` owns grant it (owner login compared casefolded) -> a set: each
|
|
356
|
+
record's `workspaces`, and every workspace the host serves (served_workspaces of `reg`, else registry.json) for a
|
|
357
|
+
record that lists none — empty or absent (DEF-1852; the module docstring has the rule). The records
|
|
358
|
+
keys.load_members accepts, as enrold.member_logins reads them."""
|
|
359
|
+
members, _problems = K.load_members()
|
|
360
|
+
folded = login.casefold()
|
|
361
|
+
found, every = set(), False
|
|
362
|
+
for member in members:
|
|
363
|
+
owner = member.get('owner')
|
|
364
|
+
if isinstance(owner, dict) and owner.get('login', '').casefold() == folded:
|
|
365
|
+
listed = _listed(member)
|
|
366
|
+
found |= listed
|
|
367
|
+
every = every or not listed
|
|
368
|
+
if every:
|
|
369
|
+
found |= served_workspaces(registry() if reg is None else reg)
|
|
370
|
+
return found
|
|
371
|
+
|
|
372
|
+
|
|
373
|
+
def member_grants(reg):
|
|
374
|
+
"""{login casefolded: (the login as its member record spells it, the workspaces its records grant)} for every
|
|
375
|
+
member at once (member_workspaces' rule, `reg` the registry): the pick-up's view of who may receive a document."""
|
|
376
|
+
members, _problems = K.load_members()
|
|
377
|
+
served = served_workspaces(reg)
|
|
378
|
+
out = {}
|
|
379
|
+
for member in members:
|
|
380
|
+
owner = member.get('owner')
|
|
381
|
+
login = owner.get('login') if isinstance(owner, dict) else None
|
|
382
|
+
if not isinstance(login, str):
|
|
383
|
+
continue
|
|
384
|
+
spelling, granted = out.get(login.casefold(), (login, set()))
|
|
385
|
+
out[login.casefold()] = (spelling, granted | (_listed(member) or served))
|
|
386
|
+
return out
|
|
387
|
+
|
|
388
|
+
|
|
389
|
+
def in_own_store(meta, login):
|
|
390
|
+
"""Whether the document `meta` describes sits in `login`'s own store: docs/<stem(login)>/<ws>/<lane>/<id> is a
|
|
391
|
+
real directory (never through a link)."""
|
|
392
|
+
try:
|
|
393
|
+
stem = D.login_stem(login)
|
|
394
|
+
except L.LanesError:
|
|
395
|
+
return False
|
|
396
|
+
try:
|
|
397
|
+
info = os.lstat(D.root() / stem / meta['ws'] / meta['lane'] / meta['id'])
|
|
398
|
+
except (OSError, KeyError, TypeError):
|
|
399
|
+
return False
|
|
400
|
+
return stat.S_ISDIR(info.st_mode)
|
|
401
|
+
|
|
402
|
+
|
|
403
|
+
def visible(meta, owner, login, workspaces):
|
|
404
|
+
"""docsd's visibility rule (see the module docstring): the caller's own store; else `for` names the caller; else
|
|
405
|
+
(`for` empty) a member record of the caller lists the document's workspace."""
|
|
406
|
+
if owner:
|
|
407
|
+
return True
|
|
408
|
+
fors = meta.get('for') or []
|
|
409
|
+
if fors:
|
|
410
|
+
folded = login.casefold()
|
|
411
|
+
return any(isinstance(entry, str) and entry.casefold() == folded for entry in fors)
|
|
412
|
+
return meta.get('ws') in workspaces
|
|
413
|
+
|
|
414
|
+
|
|
415
|
+
def visible_docs(login, workspaces):
|
|
416
|
+
"""The metas `login` may see, newest first."""
|
|
417
|
+
return [meta for meta in D.list_for(login) if visible(meta, in_own_store(meta, login), login, workspaces)]
|
|
418
|
+
|
|
419
|
+
|
|
420
|
+
def find_doc(doc_id, login, workspaces):
|
|
421
|
+
"""(meta, (stem, ws, lane, id, name)) of a document the caller may see, else None."""
|
|
422
|
+
D.check_id(doc_id)
|
|
423
|
+
found = D.load(doc_id, login)
|
|
424
|
+
if found is None:
|
|
425
|
+
return None
|
|
426
|
+
meta, path = found
|
|
427
|
+
parts = path.relative_to(D.root()).parts
|
|
428
|
+
if len(parts) != 5:
|
|
429
|
+
return None
|
|
430
|
+
try:
|
|
431
|
+
owner = parts[0] == D.login_stem(login)
|
|
432
|
+
except L.LanesError:
|
|
433
|
+
owner = False
|
|
434
|
+
return (meta, parts) if visible(meta, owner, login, workspaces) else None
|
|
435
|
+
|
|
436
|
+
|
|
437
|
+
def open_doc(meta, parts):
|
|
438
|
+
"""A read-only fd of the document file: every directory component a real directory, the file never a link, never a
|
|
439
|
+
FIFO, a regular file of the recorded size. Raises OSError otherwise. The caller closes it."""
|
|
440
|
+
directory = L.open_real_dir(D.root(), parts[:4])
|
|
441
|
+
try:
|
|
442
|
+
fd = os.open(parts[4], os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK | getattr(os, 'O_CLOEXEC', 0),
|
|
443
|
+
dir_fd=directory)
|
|
444
|
+
finally:
|
|
445
|
+
os.close(directory)
|
|
446
|
+
try:
|
|
447
|
+
info = os.fstat(fd)
|
|
448
|
+
if not stat.S_ISREG(info.st_mode) or info.st_size != D.stored_file(meta)[1]:
|
|
449
|
+
raise OSError('not the recorded document')
|
|
450
|
+
except BaseException:
|
|
451
|
+
os.close(fd)
|
|
452
|
+
raise
|
|
453
|
+
return fd
|
|
454
|
+
|
|
455
|
+
|
|
456
|
+
def read_receipt(parts):
|
|
457
|
+
"""receipt.json's bytes beside the document, or None (absent, a link, not a regular file, too large)."""
|
|
458
|
+
try:
|
|
459
|
+
directory = L.open_real_dir(D.root(), parts[:4])
|
|
460
|
+
except (OSError, L.LanesError):
|
|
461
|
+
return None
|
|
462
|
+
try:
|
|
463
|
+
return L.read_regular_at(directory, D.RECEIPT, D.MAX_META)
|
|
464
|
+
finally:
|
|
465
|
+
os.close(directory)
|
|
466
|
+
|
|
467
|
+
|
|
468
|
+
def read_all(fd, limit):
|
|
469
|
+
chunks, size = [], 0
|
|
470
|
+
while size <= limit:
|
|
471
|
+
chunk = os.read(fd, min(CHUNK, limit + 1 - size))
|
|
472
|
+
if not chunk:
|
|
473
|
+
break
|
|
474
|
+
chunks.append(chunk)
|
|
475
|
+
size += len(chunk)
|
|
476
|
+
return b''.join(chunks)
|
|
477
|
+
|
|
478
|
+
|
|
479
|
+
def registry():
|
|
480
|
+
"""registry.json, or None when there is none to read."""
|
|
481
|
+
try:
|
|
482
|
+
return L.load_registry()
|
|
483
|
+
except (L.LanesError, OSError, ValueError):
|
|
484
|
+
return None
|
|
485
|
+
|
|
486
|
+
|
|
487
|
+
def upload_targets(workspaces, reg):
|
|
488
|
+
"""[(ws id, ws title, [(lane id, lane name)])] — the registry's lanes of the caller's workspaces, catalog order."""
|
|
489
|
+
targets = []
|
|
490
|
+
for workspace in (reg or {}).get('workspaces') or []:
|
|
491
|
+
if not isinstance(workspace, dict) or workspace.get('id') not in workspaces:
|
|
492
|
+
continue
|
|
493
|
+
lanes = [(lane['id'], lane.get('name') or lane['id']) for lane in workspace.get('lanes') or []
|
|
494
|
+
if isinstance(lane, dict) and isinstance(lane.get('id'), str)]
|
|
495
|
+
if lanes:
|
|
496
|
+
targets.append((workspace['id'], workspace.get('title') or workspace['id'], lanes))
|
|
497
|
+
return targets
|
|
498
|
+
|
|
499
|
+
|
|
500
|
+
def index_filter(query, workspaces, docs, targets):
|
|
501
|
+
"""(ws, lane) a `GET /?ws=…&lane=…` query names (None for one left out). Known to the caller: a workspace its member
|
|
502
|
+
records list or a visible document is in; a lane of the registry in such a workspace (`targets`) or one a visible
|
|
503
|
+
document is in — and of the named workspace when both are given. Raises _Reply(404) for anything else (unknown,
|
|
504
|
+
repeated, empty, or more fields than a filter has); other keys are ignored."""
|
|
505
|
+
try:
|
|
506
|
+
params = urllib.parse.parse_qs(query or '', keep_blank_values=True, max_num_fields=8)
|
|
507
|
+
except ValueError:
|
|
508
|
+
raise _Reply(404, 'not-found') from None
|
|
509
|
+
picked = {}
|
|
510
|
+
for key in ('ws', 'lane'):
|
|
511
|
+
values = params.get(key)
|
|
512
|
+
if values is None:
|
|
513
|
+
picked[key] = None
|
|
514
|
+
elif len(values) != 1 or not values[0]:
|
|
515
|
+
raise _Reply(404, 'not-found')
|
|
516
|
+
else:
|
|
517
|
+
picked[key] = values[0]
|
|
518
|
+
known = {ws: set() for ws in workspaces}
|
|
519
|
+
for ws, _title, lanes in targets:
|
|
520
|
+
known.setdefault(ws, set()).update(lane for lane, _name in lanes)
|
|
521
|
+
for meta in docs:
|
|
522
|
+
known.setdefault(meta['ws'], set()).add(meta['lane'])
|
|
523
|
+
ws, lane = picked['ws'], picked['lane']
|
|
524
|
+
if ws is not None and ws not in known:
|
|
525
|
+
raise _Reply(404, 'not-found')
|
|
526
|
+
if lane is not None and lane not in (known[ws] if ws is not None else set().union(*known.values())):
|
|
527
|
+
raise _Reply(404, 'not-found')
|
|
528
|
+
return ws, lane
|
|
529
|
+
|
|
530
|
+
|
|
531
|
+
# ---------------------------------------------------------------------------------------------------- the form
|
|
532
|
+
|
|
533
|
+
class FormError(Exception):
|
|
534
|
+
def __init__(self, error):
|
|
535
|
+
super().__init__(error)
|
|
536
|
+
self.error = error
|
|
537
|
+
|
|
538
|
+
|
|
539
|
+
def form_boundary(content_type):
|
|
540
|
+
"""The boundary of a `multipart/form-data` Content-Type (read with email.message), or None."""
|
|
541
|
+
if not content_type or len(content_type) > MAX_CONTENT_TYPE or _CONTROL.search(content_type):
|
|
542
|
+
return None
|
|
543
|
+
message = email.message.Message()
|
|
544
|
+
message['Content-Type'] = content_type
|
|
545
|
+
if message.get_content_type() != 'multipart/form-data':
|
|
546
|
+
return None
|
|
547
|
+
boundary = message.get_param('boundary')
|
|
548
|
+
if not isinstance(boundary, str) or not _BOUNDARY.fullmatch(boundary):
|
|
549
|
+
return None
|
|
550
|
+
return boundary
|
|
551
|
+
|
|
552
|
+
|
|
553
|
+
def parse_form(body, boundary):
|
|
554
|
+
"""A multipart/form-data body -> (fields {name: str}, files [(filename, start, end)]) with the files as offsets into
|
|
555
|
+
`body` (no copy). The delimiters are found by a linear bytes scan (RFC 2046: CRLF "--" boundary, the last one
|
|
556
|
+
followed by "--"); each part's header block is read by email.parser (Content-Disposition form-data, its name and
|
|
557
|
+
filename — filename* included — and a Content-Transfer-Encoding of 7bit/8bit/binary or none). Parsing the whole
|
|
558
|
+
body with email.parser instead held about 13 times a 32 MiB body in memory (measured, see RUNBOOK). A part with an
|
|
559
|
+
empty filename and no bytes (a file input left empty) is skipped. Raises FormError('bad-request')."""
|
|
560
|
+
delimiter = b'--' + boundary.encode('ascii')
|
|
561
|
+
if body.startswith(delimiter):
|
|
562
|
+
at = len(delimiter)
|
|
563
|
+
else: # a preamble is allowed; browsers send none
|
|
564
|
+
found = body.find(b'\r\n' + delimiter)
|
|
565
|
+
if found < 0:
|
|
566
|
+
raise FormError('bad-request')
|
|
567
|
+
at = found + 2 + len(delimiter)
|
|
568
|
+
fields, files, parts = {}, [], 0
|
|
569
|
+
parser = email.parser.BytesHeaderParser(policy=email.policy.HTTP)
|
|
570
|
+
while True:
|
|
571
|
+
if body[at:at + 2] == b'--': # the close delimiter: the epilogue is ignored
|
|
572
|
+
return fields, files
|
|
573
|
+
line_end = body.find(b'\r\n', at)
|
|
574
|
+
if line_end < 0 or body[at:line_end].strip(b' \t'): # transport padding only, then CRLF
|
|
575
|
+
raise FormError('bad-request')
|
|
576
|
+
start = line_end + 2
|
|
577
|
+
end = body.find(b'\r\n' + delimiter, start)
|
|
578
|
+
if end < 0:
|
|
579
|
+
raise FormError('bad-request')
|
|
580
|
+
parts += 1
|
|
581
|
+
if parts > MAX_PARTS:
|
|
582
|
+
raise FormError('bad-request')
|
|
583
|
+
if body[start:start + 2] == b'\r\n':
|
|
584
|
+
header, content = b'', start + 2
|
|
585
|
+
else:
|
|
586
|
+
header_end = body.find(b'\r\n\r\n', start, end)
|
|
587
|
+
if header_end < 0 or header_end - start > MAX_PART_HEADER:
|
|
588
|
+
raise FormError('bad-request')
|
|
589
|
+
header, content = body[start:header_end + 2], header_end + 4
|
|
590
|
+
part = parser.parsebytes(header)
|
|
591
|
+
if part.get_content_disposition() != 'form-data':
|
|
592
|
+
raise FormError('bad-request')
|
|
593
|
+
name = part.get_param('name', header='content-disposition')
|
|
594
|
+
encoding = (part.get('Content-Transfer-Encoding') or '7bit').strip().lower()
|
|
595
|
+
if not isinstance(name, str) or encoding not in _CTE_OK:
|
|
596
|
+
raise FormError('bad-request')
|
|
597
|
+
filename = part.get_filename()
|
|
598
|
+
if filename is not None or part.get_param('filename', header='content-disposition') is not None:
|
|
599
|
+
if filename or end > content:
|
|
600
|
+
files.append((filename or '', content, end))
|
|
601
|
+
else:
|
|
602
|
+
if name in fields or end - content > MAX_FIELD:
|
|
603
|
+
raise FormError('bad-request')
|
|
604
|
+
try:
|
|
605
|
+
fields[name] = body[content:end].decode('utf-8')
|
|
606
|
+
except UnicodeDecodeError:
|
|
607
|
+
raise FormError('bad-request') from None
|
|
608
|
+
at = end + 2 + len(delimiter)
|
|
609
|
+
|
|
610
|
+
|
|
611
|
+
class _Slice:
|
|
612
|
+
"""A reader over body[start:end] (docs_store.intake reads it in chunks; nothing is copied whole)."""
|
|
613
|
+
|
|
614
|
+
def __init__(self, data, start, end):
|
|
615
|
+
self._view, self._at = memoryview(data)[start:end], 0
|
|
616
|
+
|
|
617
|
+
def read(self, size=-1):
|
|
618
|
+
if size is None or size < 0:
|
|
619
|
+
size = len(self._view) - self._at
|
|
620
|
+
chunk = self._view[self._at:self._at + size]
|
|
621
|
+
self._at += len(chunk)
|
|
622
|
+
return bytes(chunk)
|
|
623
|
+
|
|
624
|
+
|
|
625
|
+
def upload_target(fields, workspaces, reg):
|
|
626
|
+
"""(ws, lane) for a form's `lane` (and optional `ws`), validated against the registry and the caller's workspaces.
|
|
627
|
+
Raises FormError: no-lane, unknown-lane, unknown-workspace, registry-unavailable."""
|
|
628
|
+
lane = fields.get('lane')
|
|
629
|
+
if not lane:
|
|
630
|
+
raise FormError('no-lane')
|
|
631
|
+
try:
|
|
632
|
+
users.check_lane(lane)
|
|
633
|
+
except L.LanesError:
|
|
634
|
+
raise FormError('unknown-lane') from None
|
|
635
|
+
if reg is None:
|
|
636
|
+
raise FormError('registry-unavailable')
|
|
637
|
+
home = next((workspace.get('id') for workspace in reg.get('workspaces') or [] if isinstance(workspace, dict)
|
|
638
|
+
for entry in workspace.get('lanes') or [] if isinstance(entry, dict) and entry.get('id') == lane),
|
|
639
|
+
None)
|
|
640
|
+
if home is None:
|
|
641
|
+
raise FormError('unknown-lane')
|
|
642
|
+
named = fields.get('ws')
|
|
643
|
+
if (named is not None and named != home) or home not in workspaces:
|
|
644
|
+
raise FormError('unknown-workspace')
|
|
645
|
+
return home, lane
|
|
646
|
+
|
|
647
|
+
|
|
648
|
+
# ------------------------------------------------------------------------------------------------------- pages
|
|
649
|
+
|
|
650
|
+
def page(title, content, up='', assets=()):
|
|
651
|
+
"""The page shell. `up` is the relative way back to the docs root ('' at /, '../' at /d/<id>): every reference is
|
|
652
|
+
relative, because Caddy serves this under /lanes-docs/ and strips the prefix. `assets` names extra stylesheet +
|
|
653
|
+
script pairs of web/ (the index's drop area: 'upload')."""
|
|
654
|
+
home = up or './'
|
|
655
|
+
extra = ''.join(f'<link rel="stylesheet" href="{up}{name}.css"><script src="{up}{name}.js" defer></script>'
|
|
656
|
+
for name in assets)
|
|
657
|
+
return ('<!doctype html><html lang="en"><head><meta charset="utf-8">'
|
|
658
|
+
'<meta name="viewport" content="width=device-width, initial-scale=1">'
|
|
659
|
+
'<meta name="referrer" content="no-referrer">'
|
|
660
|
+
f'<title>{esc(title)}</title>'
|
|
661
|
+
f'<link rel="stylesheet" href="{up}docs.css">'
|
|
662
|
+
f'<script src="{up}docs.js" defer></script>{extra}</head>'
|
|
663
|
+
f'<body class="lanes-docs"><header class="docs-top"><a class="docs-home" href="{home}">Docs</a></header>'
|
|
664
|
+
f'<main>{content}</main></body></html>').encode('utf-8')
|
|
665
|
+
|
|
666
|
+
|
|
667
|
+
def message_page(status, word):
|
|
668
|
+
sentence = SENTENCES.get(word, word)
|
|
669
|
+
return page('Docs', f'<h1>Docs</h1><p class="docs-note" data-error="{esc(word)}">{esc(sentence)}</p>'
|
|
670
|
+
f'<p class="docs-muted">{status} {esc(word)}</p>')
|
|
671
|
+
|
|
672
|
+
|
|
673
|
+
def _doc_row(meta):
|
|
674
|
+
label = meta.get('title') or meta['name']
|
|
675
|
+
return (f'<li class="docs-doc" data-kind="{esc(meta["kind"])}"><a href="d/{esc(meta["id"])}">{esc(label)}</a> '
|
|
676
|
+
f'<span class="docs-meta">{esc(meta["kind"])} · {esc(human_bytes(meta["bytes"]))} · '
|
|
677
|
+
f'<time>{esc(meta["at"])}</time></span></li>')
|
|
678
|
+
|
|
679
|
+
|
|
680
|
+
def index_page(login, docs, reg, targets, ws=None, lane=None):
|
|
681
|
+
"""The index: `docs` (visible, newest first) narrowed to the filter `ws`/`lane` (index_filter checked them)."""
|
|
682
|
+
names = {}
|
|
683
|
+
titles = {}
|
|
684
|
+
for workspace in (reg or {}).get('workspaces') or []:
|
|
685
|
+
if isinstance(workspace, dict) and isinstance(workspace.get('id'), str):
|
|
686
|
+
titles[workspace['id']] = workspace.get('title') or workspace['id']
|
|
687
|
+
for entry in workspace.get('lanes') or []:
|
|
688
|
+
if isinstance(entry, dict) and isinstance(entry.get('id'), str):
|
|
689
|
+
names[entry['id']] = entry.get('name') or entry['id']
|
|
690
|
+
docs = [meta for meta in docs if (ws is None or meta['ws'] == ws) and (lane is None or meta['lane'] == lane)]
|
|
691
|
+
grouped = {}
|
|
692
|
+
for meta in docs: # newest first, so groups come newest first too
|
|
693
|
+
grouped.setdefault(meta['ws'], {}).setdefault(meta['lane'], []).append(meta)
|
|
694
|
+
parts = [f'<h1>Docs</h1><p class="docs-who">Signed in as {esc(login)} · {len(docs)} '
|
|
695
|
+
f'document{"" if len(docs) == 1 else "s"}</p>']
|
|
696
|
+
if ws is not None or lane is not None:
|
|
697
|
+
shown = ' › '.join(esc(label) for label in (titles.get(ws, ws) if ws else None,
|
|
698
|
+
names.get(lane, lane) if lane else None) if label)
|
|
699
|
+
parts.append(f'<p class="docs-filter">Showing {shown} · <a href="./">All docs</a></p>')
|
|
700
|
+
if targets:
|
|
701
|
+
options = ''.join(f'<optgroup label="{esc(title)}">' + ''.join(
|
|
702
|
+
f'<option value="{esc(lane_id)}"{" selected" if lane_id == lane else ""}>{esc(name)}</option>'
|
|
703
|
+
for lane_id, name in lanes) + '</optgroup>' for _ws, title, lanes in targets)
|
|
704
|
+
# The drop area (W5): a plain form that works without a script — the label wraps a multi-file input, so a tap
|
|
705
|
+
# opens the picker and a drop onto the input takes the files —, and better with upload.js (drag and drop
|
|
706
|
+
# onto the whole area, a size check, progress, one receipt per file in .docs-sent).
|
|
707
|
+
parts.append(f'<form class="docs-upload" method="post" action="up" enctype="multipart/form-data" '
|
|
708
|
+
f'data-max="{MAX_BODY}">'
|
|
709
|
+
f'<label>Lane <select name="lane" required>{options}</select></label>'
|
|
710
|
+
'<label class="docs-drop">Drop files here, or tap to choose'
|
|
711
|
+
'<input type="file" name="file" multiple required></label>'
|
|
712
|
+
'<button type="submit">Upload</button>'
|
|
713
|
+
f'<p class="docs-muted">Up to {human_bytes(MAX_BODY)} at once. Each file lands in the '
|
|
714
|
+
'workspace\'s uploads/ folder, for the lane\'s agent, with a sha256 receipt here.</p>'
|
|
715
|
+
'<ol class="docs-sent" aria-live="polite"></ol></form>')
|
|
716
|
+
else:
|
|
717
|
+
parts.append('<p class="docs-note">Uploads go to a lane of a workspace your member record lists, and yours '
|
|
718
|
+
'lists none here.</p>')
|
|
719
|
+
if not docs:
|
|
720
|
+
parts.append('<p class="docs-empty">No documents yet. An agent publishes one from a lane; an upload lands '
|
|
721
|
+
'here with its receipt.</p>')
|
|
722
|
+
for ws_id, lanes in grouped.items():
|
|
723
|
+
total = sum(len(metas) for metas in lanes.values())
|
|
724
|
+
parts.append(f'<section class="docs-ws" data-ws="{esc(ws_id)}"><h2>{esc(titles.get(ws_id, ws_id))} '
|
|
725
|
+
f'<span class="docs-count">{total}</span></h2>')
|
|
726
|
+
for lane_id, metas in lanes.items():
|
|
727
|
+
parts.append(f'<h3 data-lane="{esc(lane_id)}">{esc(names.get(lane_id, lane_id))} '
|
|
728
|
+
f'<span class="docs-count">{len(metas)}</span></h3><ul class="docs-list">'
|
|
729
|
+
+ ''.join(_doc_row(meta) for meta in metas) + '</ul>')
|
|
730
|
+
parts.append('</section>')
|
|
731
|
+
return page('Docs', ''.join(parts), assets=('upload',) if targets else ())
|
|
732
|
+
|
|
733
|
+
|
|
734
|
+
def _facts(meta):
|
|
735
|
+
rows = (('kind', meta['kind']), ('name', meta['name']), ('size', human_bytes(meta['bytes'])),
|
|
736
|
+
('workspace', meta['ws']), ('lane', meta['lane']), ('at', meta['at']), ('by', meta.get('by') or '-'),
|
|
737
|
+
('source', meta['source']), ('sha256', meta['sha256']))
|
|
738
|
+
return '<dl class="docs-facts">' + ''.join(f'<dt>{esc(key)}</dt><dd>{esc(value)}</dd>' for key, value in rows) \
|
|
739
|
+
+ '</dl>'
|
|
740
|
+
|
|
741
|
+
|
|
742
|
+
def view_page(meta, body=None, locked=False):
|
|
743
|
+
"""A document's page. `locked`: an encrypted document this service cannot open (no identity): its facts only."""
|
|
744
|
+
doc_id = esc(meta['id'])
|
|
745
|
+
title = meta.get('title') or meta['name']
|
|
746
|
+
links = (f'<p class="docs-links"><a href="{doc_id}/raw" download>Download</a> · '
|
|
747
|
+
f'<a href="../receipt/{doc_id}.json">Receipt</a></p>')
|
|
748
|
+
if locked:
|
|
749
|
+
content = f'<p class="docs-note" data-error="encrypted">{esc(SENTENCES["encrypted"])}</p>'
|
|
750
|
+
elif meta['kind'] == 'md':
|
|
751
|
+
if body is None:
|
|
752
|
+
content = (f'<p class="docs-note">This Markdown is larger than {human_bytes(mdlite.MAX_BYTES)} and is not '
|
|
753
|
+
'rendered here; download it.</p>')
|
|
754
|
+
else:
|
|
755
|
+
content = f'<article class="docs-md">{body}</article>'
|
|
756
|
+
elif meta['kind'] in ARTIFACT_TYPES:
|
|
757
|
+
content = (f'<p class="docs-artifact"><a href="{doc_id}/artifact" rel="noopener noreferrer">Open the '
|
|
758
|
+
f'{esc(meta["kind"].upper())} artifact</a></p><p class="docs-muted">It opens on its own page in a '
|
|
759
|
+
'sandbox: its scripts run, but cannot reach this site or your lanes.</p>')
|
|
760
|
+
else:
|
|
761
|
+
content = ''
|
|
762
|
+
return page(title, f'<h1>{esc(title)}</h1>{links}{content}{_facts(meta)}', up='../')
|
|
763
|
+
|
|
764
|
+
|
|
765
|
+
def receipts_page(receipts, error=None):
|
|
766
|
+
rows = ''.join(f'<li><a href="d/{esc(r["id"])}">{esc(r["name"])}</a> <span class="docs-meta">'
|
|
767
|
+
f'{esc(human_bytes(r["bytes"]))} · sha256 <code>{esc(r["sha256"])}</code> · in the lane as '
|
|
768
|
+
f'<code>{esc(r["landed"])}</code></span></li>' for r in receipts)
|
|
769
|
+
note = f'<p class="docs-note" data-error="{esc(error)}">{esc(SENTENCES.get(error, error))}</p>' if error else ''
|
|
770
|
+
return page('Uploaded', f'<h1>Uploaded</h1>{note}<ul class="docs-receipts">{rows}</ul>'
|
|
771
|
+
'<p><a href="./">Back to the docs</a></p>')
|
|
772
|
+
|
|
773
|
+
|
|
774
|
+
# ----------------------------------------------------------------------------------------------------- service
|
|
775
|
+
|
|
776
|
+
def fetch_site_ok(headers, method):
|
|
777
|
+
"""The gate's `cross-site` rule (the module docstring): `Sec-Fetch-Site` same-origin or none, or a GET/HEAD
|
|
778
|
+
top-level navigation (`Sec-Fetch-Mode: navigate`, `Sec-Fetch-Dest: document`) from any site. A POST — an upload —
|
|
779
|
+
is never a navigation here."""
|
|
780
|
+
if headers.get('Sec-Fetch-Site') in FETCH_SITES:
|
|
781
|
+
return True
|
|
782
|
+
return (method in NAVIGATION_METHODS and headers.get('Sec-Fetch-Mode') == 'navigate'
|
|
783
|
+
and headers.get('Sec-Fetch-Dest') == 'document')
|
|
784
|
+
|
|
785
|
+
|
|
786
|
+
class RenderCache:
|
|
787
|
+
"""Rendered Markdown by the document's sha256 (the same bytes render the same, mdlite is deterministic), least
|
|
788
|
+
recently used out first: at most RENDER_CACHE_BYTES in all, nothing larger than RENDER_CACHE_ENTRY kept. A value
|
|
789
|
+
of None is a document too large to render once its bytes are decoded."""
|
|
790
|
+
|
|
791
|
+
def __init__(self):
|
|
792
|
+
self._lock = threading.Lock()
|
|
793
|
+
self._entries = collections.OrderedDict()
|
|
794
|
+
self.size = 0
|
|
795
|
+
|
|
796
|
+
def get(self, key):
|
|
797
|
+
"""(True, the render) when `key` is kept (it becomes the most recent), else (False, None)."""
|
|
798
|
+
with self._lock:
|
|
799
|
+
if key not in self._entries:
|
|
800
|
+
return False, None
|
|
801
|
+
self._entries.move_to_end(key)
|
|
802
|
+
return True, self._entries[key][0]
|
|
803
|
+
|
|
804
|
+
def put(self, key, value):
|
|
805
|
+
size = sys.getsizeof(value)
|
|
806
|
+
if size > RENDER_CACHE_ENTRY:
|
|
807
|
+
return
|
|
808
|
+
with self._lock:
|
|
809
|
+
if key in self._entries:
|
|
810
|
+
return
|
|
811
|
+
self._entries[key] = (value, size)
|
|
812
|
+
self.size += size
|
|
813
|
+
while self.size > RENDER_CACHE_BYTES and self._entries:
|
|
814
|
+
_key, (_value, gone) = self._entries.popitem(last=False)
|
|
815
|
+
self.size -= gone
|
|
816
|
+
|
|
817
|
+
def __len__(self):
|
|
818
|
+
return len(self._entries)
|
|
819
|
+
|
|
820
|
+
|
|
821
|
+
class Service:
|
|
822
|
+
"""Process-wide state: the domain gate, the limiters, the upload/decrypt slots, the render slots and cache, and the
|
|
823
|
+
coalescing of refusals in docs.log."""
|
|
824
|
+
|
|
825
|
+
def __init__(self, domains=None, clock=time.time):
|
|
826
|
+
self.domains = tuple(domains) if domains else E.DEFAULT_DOMAINS
|
|
827
|
+
self.clock = clock
|
|
828
|
+
self._lock = threading.Lock()
|
|
829
|
+
self._uploads = {}
|
|
830
|
+
self._views = {} # login casefolded -> [window start, requests in it]
|
|
831
|
+
self._holders = set() # logins casefolded holding an upload slot (one each)
|
|
832
|
+
self._denied = {} # (word, source) -> [window start, refusals folded since, last login]
|
|
833
|
+
self._swept = float('-inf')
|
|
834
|
+
self.slots = threading.BoundedSemaphore(UPLOAD_SLOTS)
|
|
835
|
+
self.renders = threading.BoundedSemaphore(RENDER_SLOTS)
|
|
836
|
+
self.rendered = RenderCache()
|
|
837
|
+
self._pin_problem = None
|
|
838
|
+
|
|
839
|
+
@classmethod
|
|
840
|
+
def from_environ(cls, environ):
|
|
841
|
+
return cls(domains=E.parse_domains(environ.get('LANES_DOCS_DOMAINS') or environ.get('LANES_ENROL_DOMAINS')))
|
|
842
|
+
|
|
843
|
+
domain_ok = E.Service.domain_ok # enrold's rule over self.domains, the same function
|
|
844
|
+
|
|
845
|
+
def pinned(self):
|
|
846
|
+
"""Whether the operator pinned request mode (enrold.read_mode): the proof the Serve header check ran."""
|
|
847
|
+
mode, problem = E.read_mode()
|
|
848
|
+
if problem and problem != self._pin_problem:
|
|
849
|
+
print(f'lanes-docsd: {problem}: every request is refused until `lanes enrol mode request` writes it again',
|
|
850
|
+
file=sys.stderr, flush=True)
|
|
851
|
+
self._pin_problem = problem
|
|
852
|
+
return mode == 'request'
|
|
853
|
+
|
|
854
|
+
def gate(self, headers, peer, method='GET'):
|
|
855
|
+
"""(caller, None) when the request may be served, else (caller, the refusal's word). See the module docstring
|
|
856
|
+
for the order."""
|
|
857
|
+
caller = E.caller_of(headers)
|
|
858
|
+
if not E.through_serve(headers.get('X-Forwarded-For'), peer):
|
|
859
|
+
return caller, 'not-through-serve'
|
|
860
|
+
if not caller.present:
|
|
861
|
+
return caller, 'no-identity'
|
|
862
|
+
if not caller.login:
|
|
863
|
+
return caller, 'invalid-login'
|
|
864
|
+
if not self.pinned():
|
|
865
|
+
return caller, 'not-pinned'
|
|
866
|
+
if caller.login.casefold() not in E.member_logins():
|
|
867
|
+
return caller, 'not-a-member'
|
|
868
|
+
if not self.domain_ok(caller.login):
|
|
869
|
+
return caller, 'domain-not-allowed'
|
|
870
|
+
if not fetch_site_ok(headers, method):
|
|
871
|
+
return caller, 'cross-site'
|
|
872
|
+
return caller, None
|
|
873
|
+
|
|
874
|
+
def attempt(self, login):
|
|
875
|
+
"""0 when `login` may upload now (the attempt is counted), else the seconds to wait."""
|
|
876
|
+
now, key = self.clock(), login.casefold()
|
|
877
|
+
with self._lock:
|
|
878
|
+
recent = [at for at in self._uploads.get(key, ()) if now - at < UPLOAD_WINDOW]
|
|
879
|
+
if len(recent) >= UPLOADS:
|
|
880
|
+
self._uploads[key] = recent
|
|
881
|
+
return int(recent[0] + UPLOAD_WINDOW - now) + 1
|
|
882
|
+
self._uploads[key] = recent + [now]
|
|
883
|
+
if len(self._uploads) > MAX_LOGINS:
|
|
884
|
+
for other in list(self._uploads):
|
|
885
|
+
kept = [at for at in self._uploads[other] if now - at < UPLOAD_WINDOW]
|
|
886
|
+
if kept:
|
|
887
|
+
self._uploads[other] = kept
|
|
888
|
+
else:
|
|
889
|
+
del self._uploads[other]
|
|
890
|
+
while len(self._uploads) > MAX_LOGINS:
|
|
891
|
+
self._uploads.pop(min(self._uploads, key=lambda name: self._uploads[name][-1]))
|
|
892
|
+
return 0
|
|
893
|
+
|
|
894
|
+
def view(self, login):
|
|
895
|
+
"""(0, False) when `login` may make another request now (it is counted: VIEWS per VIEW_WINDOW s, a fixed
|
|
896
|
+
window), else (the seconds to wait, whether this is the window's first refusal — the one docs.log line)."""
|
|
897
|
+
now, key = self.clock(), login.casefold()
|
|
898
|
+
with self._lock:
|
|
899
|
+
window = self._views.get(key)
|
|
900
|
+
if window is None or now - window[0] >= VIEW_WINDOW:
|
|
901
|
+
if window is None and len(self._views) >= MAX_LOGINS:
|
|
902
|
+
for other in [name for name, (start, _n) in self._views.items() if now - start >= VIEW_WINDOW]:
|
|
903
|
+
del self._views[other]
|
|
904
|
+
while len(self._views) >= MAX_LOGINS:
|
|
905
|
+
self._views.pop(min(self._views, key=lambda name: self._views[name][0]))
|
|
906
|
+
window = self._views[key] = [now, 0]
|
|
907
|
+
window[1] += 1
|
|
908
|
+
if window[1] <= VIEWS:
|
|
909
|
+
return 0, False
|
|
910
|
+
return int(window[0] + VIEW_WINDOW - now) + 1, window[1] == VIEWS + 1
|
|
911
|
+
|
|
912
|
+
def take_upload_slot(self, login):
|
|
913
|
+
"""Whether `login` got one of the UPLOAD_SLOTS (never a second one while it holds one: F5)."""
|
|
914
|
+
key = login.casefold()
|
|
915
|
+
with self._lock:
|
|
916
|
+
if key in self._holders or not self.slots.acquire(blocking=False):
|
|
917
|
+
return False
|
|
918
|
+
self._holders.add(key)
|
|
919
|
+
return True
|
|
920
|
+
|
|
921
|
+
def give_upload_slot(self, login):
|
|
922
|
+
with self._lock:
|
|
923
|
+
self._holders.discard(login.casefold())
|
|
924
|
+
self.slots.release()
|
|
925
|
+
|
|
926
|
+
def denied(self, login, word, source):
|
|
927
|
+
"""Log a gate refusal, coalesced (F4): the first of a (word, source) is a `denied` line; more within
|
|
928
|
+
DENIED_REPEAT_S are counted, and flush_denied writes one `denied-repeat` line per window with the count."""
|
|
929
|
+
now = self.clock()
|
|
930
|
+
shown = login[:MAX_DENIED_LOGIN] if login else None
|
|
931
|
+
lines = self._sweep(now)
|
|
932
|
+
with self._lock:
|
|
933
|
+
entry = self._denied.get((word, source))
|
|
934
|
+
if entry is not None:
|
|
935
|
+
entry[1] += 1
|
|
936
|
+
entry[2] = shown
|
|
937
|
+
else:
|
|
938
|
+
while len(self._denied) >= MAX_DENIED_SOURCES: # forget the oldest; its count is said first
|
|
939
|
+
oldest = min(self._denied, key=lambda key: self._denied[key][0])
|
|
940
|
+
start, count, last = self._denied.pop(oldest)
|
|
941
|
+
if count:
|
|
942
|
+
lines.append((last, count, oldest[0]))
|
|
943
|
+
self._denied[(word, source)] = [now, 0, shown]
|
|
944
|
+
lines.append((shown, None, word))
|
|
945
|
+
for logged, count, said in lines:
|
|
946
|
+
D.audit('denied' if count is None else 'denied-repeat', logged, None, None, count, None, said, now)
|
|
947
|
+
|
|
948
|
+
def flush_denied(self, now=None):
|
|
949
|
+
"""Write the `denied-repeat` lines whose window has passed (the pick-up's cycle calls this every few s)."""
|
|
950
|
+
now = self.clock() if now is None else now
|
|
951
|
+
for logged, count, said in self._sweep(now, force=True):
|
|
952
|
+
D.audit('denied-repeat', logged, None, None, count, None, said, now)
|
|
953
|
+
|
|
954
|
+
def _sweep(self, now, force=False):
|
|
955
|
+
"""The repeat lines due at `now`, at most once a second unless `force`: a window with refusals folded into it
|
|
956
|
+
is said and restarted; one without is forgotten (the next refusal is a `denied` line again)."""
|
|
957
|
+
lines = []
|
|
958
|
+
with self._lock:
|
|
959
|
+
if not force and now - self._swept < 1:
|
|
960
|
+
return lines
|
|
961
|
+
self._swept = now
|
|
962
|
+
for key in list(self._denied):
|
|
963
|
+
start, count, last = self._denied[key]
|
|
964
|
+
if now - start < DENIED_REPEAT_S:
|
|
965
|
+
continue
|
|
966
|
+
if count:
|
|
967
|
+
lines.append((last, count, key[0]))
|
|
968
|
+
self._denied[key] = [now, 0, last]
|
|
969
|
+
else:
|
|
970
|
+
del self._denied[key]
|
|
971
|
+
return lines
|
|
972
|
+
|
|
973
|
+
|
|
974
|
+
# -------------------------------------------------------------------------------------------------------- HTTP
|
|
975
|
+
|
|
976
|
+
class _Reply(Exception):
|
|
977
|
+
def __init__(self, status, error, headers=()):
|
|
978
|
+
super().__init__(error)
|
|
979
|
+
self.status, self.error, self.headers = status, error, tuple(headers)
|
|
980
|
+
|
|
981
|
+
|
|
982
|
+
def resolve(path):
|
|
983
|
+
"""(route, the id or file name it names, or None) for a request path, else (None, None)."""
|
|
984
|
+
for route, pattern in ROUTES:
|
|
985
|
+
found = pattern.fullmatch(path)
|
|
986
|
+
if found:
|
|
987
|
+
return route, (found.group(1) if found.groups() else None)
|
|
988
|
+
return None, None
|
|
989
|
+
|
|
990
|
+
|
|
991
|
+
class Handler(BaseHTTPRequestHandler):
|
|
992
|
+
server_version = 'lanes-docsd'
|
|
993
|
+
timeout = 30
|
|
994
|
+
|
|
995
|
+
def version_string(self):
|
|
996
|
+
return self.server_version
|
|
997
|
+
|
|
998
|
+
def log_message(self, format, *args): # no access log: docs.log is the record, the journal the errors
|
|
999
|
+
pass
|
|
1000
|
+
|
|
1001
|
+
def do_GET(self):
|
|
1002
|
+
self._route('GET')
|
|
1003
|
+
|
|
1004
|
+
def do_POST(self):
|
|
1005
|
+
self._route('POST')
|
|
1006
|
+
|
|
1007
|
+
def do_HEAD(self):
|
|
1008
|
+
self._route('HEAD')
|
|
1009
|
+
|
|
1010
|
+
def do_PUT(self):
|
|
1011
|
+
self._route('PUT')
|
|
1012
|
+
|
|
1013
|
+
def do_DELETE(self):
|
|
1014
|
+
self._route('DELETE')
|
|
1015
|
+
|
|
1016
|
+
def do_PATCH(self):
|
|
1017
|
+
self._route('PATCH')
|
|
1018
|
+
|
|
1019
|
+
def do_OPTIONS(self):
|
|
1020
|
+
self._route('OPTIONS')
|
|
1021
|
+
|
|
1022
|
+
# ------------------------------------------------------------------------------------------- plumbing
|
|
1023
|
+
|
|
1024
|
+
def _headers(self, status, content_type, length, headers, policy, cache):
|
|
1025
|
+
if self.command in ('POST', 'PUT', 'PATCH') and not self._body_read:
|
|
1026
|
+
self._discard() # an unread body would reset the client's connection (DEF-1854)
|
|
1027
|
+
self.send_response(status)
|
|
1028
|
+
self.send_header('Content-Type', content_type)
|
|
1029
|
+
self.send_header('Content-Length', str(length))
|
|
1030
|
+
self.send_header('Cache-Control', cache)
|
|
1031
|
+
self.send_header('X-Content-Type-Options', 'nosniff')
|
|
1032
|
+
self.send_header('X-Frame-Options', 'DENY')
|
|
1033
|
+
self.send_header('Referrer-Policy', 'no-referrer')
|
|
1034
|
+
self.send_header('Content-Security-Policy', policy)
|
|
1035
|
+
for name, value in headers:
|
|
1036
|
+
self.send_header(name, value)
|
|
1037
|
+
if self._discarded:
|
|
1038
|
+
self.send_header('Connection', 'close')
|
|
1039
|
+
self.end_headers()
|
|
1040
|
+
|
|
1041
|
+
def _send(self, status, body, content_type, headers=(), policy=DATA_POLICY, cache='no-store'):
|
|
1042
|
+
self._headers(status, content_type, len(body), headers, policy, cache)
|
|
1043
|
+
if self.command != 'HEAD':
|
|
1044
|
+
self.wfile.write(body)
|
|
1045
|
+
self._linger()
|
|
1046
|
+
|
|
1047
|
+
def _send_fd(self, fd, size, content_type, headers=(), policy=DATA_POLICY):
|
|
1048
|
+
self._headers(200, content_type, size, headers, policy, 'no-store')
|
|
1049
|
+
left = size
|
|
1050
|
+
while left > 0:
|
|
1051
|
+
chunk = os.read(fd, min(CHUNK, left))
|
|
1052
|
+
if not chunk:
|
|
1053
|
+
break # shrank under us: the connection closes short, never padded
|
|
1054
|
+
self.wfile.write(chunk)
|
|
1055
|
+
left -= len(chunk)
|
|
1056
|
+
if left:
|
|
1057
|
+
self.close_connection = True
|
|
1058
|
+
|
|
1059
|
+
def _page(self, status, body, headers=()):
|
|
1060
|
+
self._send(status, body, 'text/html; charset=utf-8', headers, PAGE_POLICY)
|
|
1061
|
+
|
|
1062
|
+
def _json(self, status, payload, headers=()):
|
|
1063
|
+
self._send(status, json.dumps(payload).encode('utf-8'), 'application/json; charset=utf-8', headers)
|
|
1064
|
+
|
|
1065
|
+
def _error(self, status, word, headers=()):
|
|
1066
|
+
if wants_html(self.headers):
|
|
1067
|
+
return self._page(status, message_page(status, word), headers)
|
|
1068
|
+
return self._json(status, {'error': word}, headers)
|
|
1069
|
+
|
|
1070
|
+
def _discard(self):
|
|
1071
|
+
"""Read an unread request body away BEFORE a refusal is sent (DEF-1854): up to its Content-Length, at most
|
|
1072
|
+
DRAIN_LIMIT bytes. A client sends its whole body before it reads an answer; one answered and closed while it
|
|
1073
|
+
is still sending sees a reset or a broken pipe, never the refusal (W3 MEASURED that on the third of three
|
|
1074
|
+
concurrent 32 MiB uploads, when only 1 MiB was read away). What is left past DRAIN_LIMIT — or a chunked body,
|
|
1075
|
+
which is never read — is _linger's. The connection closes after the answer (`Connection: close`)."""
|
|
1076
|
+
self._body_read = self._discarded = True
|
|
1077
|
+
self.close_connection = True
|
|
1078
|
+
if self.headers.get('Transfer-Encoding') is not None:
|
|
1079
|
+
self._unread = DRAIN_LIMIT
|
|
1080
|
+
return
|
|
1081
|
+
try:
|
|
1082
|
+
declared = max(int(self.headers.get('Content-Length') or 0), 0)
|
|
1083
|
+
except ValueError:
|
|
1084
|
+
declared = 0
|
|
1085
|
+
drained, target = 0, min(declared, DRAIN_LIMIT)
|
|
1086
|
+
try:
|
|
1087
|
+
while drained < target:
|
|
1088
|
+
chunk = self.rfile.read(min(target - drained, CHUNK))
|
|
1089
|
+
if not chunk: # the client stopped sending: nothing more will come
|
|
1090
|
+
self._unread = 0
|
|
1091
|
+
return
|
|
1092
|
+
drained += len(chunk)
|
|
1093
|
+
except OSError:
|
|
1094
|
+
pass # a read timed out: what is left is _linger's
|
|
1095
|
+
self._unread = declared - drained
|
|
1096
|
+
|
|
1097
|
+
def _linger(self):
|
|
1098
|
+
"""After a refusal's answer, when the body is still arriving (past DRAIN_LIMIT, chunked, or a read timed out):
|
|
1099
|
+
shut the write side — the client's next read sees the whole answer and then its end — and read the rest away
|
|
1100
|
+
for at most LINGER_S seconds and DRAIN_LIMIT bytes before the socket closes, so the close does not reset the
|
|
1101
|
+
connection under an answer the client has not read yet (DEF-1854)."""
|
|
1102
|
+
if not self._unread:
|
|
1103
|
+
return
|
|
1104
|
+
self._unread = 0
|
|
1105
|
+
try:
|
|
1106
|
+
self.wfile.flush()
|
|
1107
|
+
self.connection.shutdown(socket.SHUT_WR)
|
|
1108
|
+
deadline, left = time.monotonic() + LINGER_S, DRAIN_LIMIT
|
|
1109
|
+
while left > 0:
|
|
1110
|
+
remaining = deadline - time.monotonic()
|
|
1111
|
+
if remaining <= 0:
|
|
1112
|
+
break
|
|
1113
|
+
self.connection.settimeout(remaining)
|
|
1114
|
+
chunk = self.connection.recv(min(CHUNK, left))
|
|
1115
|
+
if not chunk:
|
|
1116
|
+
break
|
|
1117
|
+
left -= len(chunk)
|
|
1118
|
+
except OSError:
|
|
1119
|
+
pass
|
|
1120
|
+
|
|
1121
|
+
def _body(self):
|
|
1122
|
+
if self.headers.get('Transfer-Encoding') is not None:
|
|
1123
|
+
self._discard() # chunked bodies are not read: answer, then _linger
|
|
1124
|
+
raise _Reply(411, 'length-required')
|
|
1125
|
+
length = self.headers.get('Content-Length')
|
|
1126
|
+
if length is None:
|
|
1127
|
+
raise _Reply(411, 'length-required')
|
|
1128
|
+
try:
|
|
1129
|
+
size = int(length)
|
|
1130
|
+
except ValueError:
|
|
1131
|
+
size = -1
|
|
1132
|
+
if size < 0:
|
|
1133
|
+
self._discard()
|
|
1134
|
+
raise _Reply(400, 'bad-request')
|
|
1135
|
+
if size > MAX_BODY:
|
|
1136
|
+
self._discard()
|
|
1137
|
+
raise _Reply(413, 'too-large')
|
|
1138
|
+
# The whole body has a deadline, not just each read (F5): a sender that trickles a byte every few seconds held
|
|
1139
|
+
# an upload slot for as long as it liked. Read into one buffer of the declared size (no second copy).
|
|
1140
|
+
data, got = bytearray(size), 0
|
|
1141
|
+
view = memoryview(data)
|
|
1142
|
+
deadline = time.monotonic() + BODY_GRACE_S + size / BODY_MIN_RATE
|
|
1143
|
+
try:
|
|
1144
|
+
while got < size:
|
|
1145
|
+
left = deadline - time.monotonic()
|
|
1146
|
+
if left <= 0:
|
|
1147
|
+
raise TimeoutError('the body did not arrive in time')
|
|
1148
|
+
self.connection.settimeout(min(left, self.timeout))
|
|
1149
|
+
read = self.rfile.readinto1(view[got:got + CHUNK])
|
|
1150
|
+
if not read:
|
|
1151
|
+
break
|
|
1152
|
+
got += read
|
|
1153
|
+
except TimeoutError:
|
|
1154
|
+
self._body_read = self._discarded = True # the rest is not waited for: answer and close
|
|
1155
|
+
self._unread = 0
|
|
1156
|
+
self.close_connection = True
|
|
1157
|
+
raise _Reply(408, 'request-timeout') from None
|
|
1158
|
+
except OSError:
|
|
1159
|
+
self._body_read = self._discarded = True # the client went away
|
|
1160
|
+
self._unread = 0
|
|
1161
|
+
self.close_connection = True
|
|
1162
|
+
raise _Reply(400, 'bad-request') from None
|
|
1163
|
+
finally:
|
|
1164
|
+
try:
|
|
1165
|
+
self.connection.settimeout(self.timeout)
|
|
1166
|
+
except OSError:
|
|
1167
|
+
pass
|
|
1168
|
+
self._body_read = True
|
|
1169
|
+
if got != size:
|
|
1170
|
+
self.close_connection = True
|
|
1171
|
+
raise _Reply(400, 'bad-request')
|
|
1172
|
+
return data
|
|
1173
|
+
|
|
1174
|
+
# --------------------------------------------------------------------------------------------- routes
|
|
1175
|
+
|
|
1176
|
+
def _route(self, method):
|
|
1177
|
+
self._body_read = self._discarded = False
|
|
1178
|
+
self._unread = 0
|
|
1179
|
+
service = self.server.service
|
|
1180
|
+
try:
|
|
1181
|
+
caller, refusal = service.gate(self.headers, self.client_address[0], method)
|
|
1182
|
+
if refusal:
|
|
1183
|
+
service.denied(caller.login, refusal,
|
|
1184
|
+
E.source_key(self.headers.get('X-Forwarded-For'), self.client_address[0]))
|
|
1185
|
+
return self._error(403, refusal)
|
|
1186
|
+
route, name = resolve(urllib.parse.urlsplit(self.path).path)
|
|
1187
|
+
if route != 'up': # every request but an upload counts against the login (F2)
|
|
1188
|
+
wait, first = service.view(caller.login)
|
|
1189
|
+
if wait:
|
|
1190
|
+
if first: # one line per login per window: `view … rate-limited`
|
|
1191
|
+
D.audit('view', caller.login, None, None, None, None, 'rate-limited')
|
|
1192
|
+
raise _Reply(429, 'rate-limited', (('Retry-After', str(wait)),))
|
|
1193
|
+
if route is None:
|
|
1194
|
+
return self._error(404, 'not-found')
|
|
1195
|
+
allowed = METHODS.get(route, ('GET',))
|
|
1196
|
+
if method not in allowed:
|
|
1197
|
+
return self._error(405, 'method-not-allowed', (('Allow', ', '.join(allowed)),))
|
|
1198
|
+
if route == 'static':
|
|
1199
|
+
return self._static(name)
|
|
1200
|
+
if route == 'up':
|
|
1201
|
+
return self._upload(service, caller)
|
|
1202
|
+
workspaces = member_workspaces(caller.login)
|
|
1203
|
+
if route == 'index':
|
|
1204
|
+
reg = registry()
|
|
1205
|
+
docs, targets = visible_docs(caller.login, workspaces), upload_targets(workspaces, reg)
|
|
1206
|
+
ws, lane = index_filter(urllib.parse.urlsplit(self.path).query, workspaces, docs, targets)
|
|
1207
|
+
return self._page(200, index_page(caller.login, docs, reg, targets, ws, lane))
|
|
1208
|
+
return self._document(route, name, caller.login, workspaces)
|
|
1209
|
+
except _Reply as reply:
|
|
1210
|
+
try:
|
|
1211
|
+
return self._error(reply.status, reply.error, reply.headers)
|
|
1212
|
+
except OSError: # the client went away before its answer (a stalled upload's)
|
|
1213
|
+
self.close_connection = True
|
|
1214
|
+
except Exception: # noqa: BLE001 — one request's bug never stops the service
|
|
1215
|
+
traceback.print_exc(file=sys.stderr)
|
|
1216
|
+
try:
|
|
1217
|
+
self._error(500, 'server-error')
|
|
1218
|
+
except OSError:
|
|
1219
|
+
pass
|
|
1220
|
+
|
|
1221
|
+
def _static(self, name):
|
|
1222
|
+
try:
|
|
1223
|
+
directory = L.open_real_dir(web_dir(), ())
|
|
1224
|
+
except OSError:
|
|
1225
|
+
return self._error(404, 'not-found')
|
|
1226
|
+
try:
|
|
1227
|
+
data = L.read_regular_at(directory, name, MAX_STATIC)
|
|
1228
|
+
finally:
|
|
1229
|
+
os.close(directory)
|
|
1230
|
+
if data is None:
|
|
1231
|
+
return self._error(404, 'not-found')
|
|
1232
|
+
return self._send(200, data, STATIC[name], cache='no-cache')
|
|
1233
|
+
|
|
1234
|
+
def _document(self, route, doc_id, login, workspaces):
|
|
1235
|
+
try:
|
|
1236
|
+
found = find_doc(doc_id, login, workspaces)
|
|
1237
|
+
except D.DocsError:
|
|
1238
|
+
found = None
|
|
1239
|
+
if found is None or (route == 'artifact' and found[0]['kind'] not in ARTIFACT_TYPES):
|
|
1240
|
+
D.audit(route, login, doc_id, found[0]['lane'] if found else None, None, None, 'not-found')
|
|
1241
|
+
return self._error(404, 'not-found')
|
|
1242
|
+
meta, parts = found
|
|
1243
|
+
if route == 'receipt':
|
|
1244
|
+
data = read_receipt(parts)
|
|
1245
|
+
if data is None:
|
|
1246
|
+
D.audit(route, login, doc_id, meta['lane'], None, None, 'not-found')
|
|
1247
|
+
return self._error(404, 'not-found')
|
|
1248
|
+
D.audit(route, login, doc_id, meta['lane'], meta['bytes'], meta['sha256'], 'ok')
|
|
1249
|
+
return self._send(200, data, 'application/json; charset=utf-8')
|
|
1250
|
+
try:
|
|
1251
|
+
fd = open_doc(meta, parts)
|
|
1252
|
+
except (OSError, L.LanesError):
|
|
1253
|
+
D.audit(route, login, doc_id, meta['lane'], None, None, 'not-found')
|
|
1254
|
+
return self._error(404, 'not-found')
|
|
1255
|
+
try:
|
|
1256
|
+
if meta.get('enc'):
|
|
1257
|
+
return self._encrypted(route, meta, fd, login)
|
|
1258
|
+
if route == 'view':
|
|
1259
|
+
body = None
|
|
1260
|
+
if meta['kind'] == 'md' and meta['bytes'] <= mdlite.MAX_BYTES:
|
|
1261
|
+
body = self._rendered(route, meta, login, lambda: read_all(fd, mdlite.MAX_BYTES))
|
|
1262
|
+
D.audit(route, login, doc_id, meta['lane'], meta['bytes'], meta['sha256'], 'ok')
|
|
1263
|
+
return self._page(200, view_page(meta, body))
|
|
1264
|
+
D.audit(route, login, doc_id, meta['lane'], meta['bytes'], meta['sha256'], 'ok')
|
|
1265
|
+
if route == 'raw':
|
|
1266
|
+
disposition = f'attachment; filename="{download_name(meta["name"])}"'
|
|
1267
|
+
return self._send_fd(fd, meta['bytes'], content_type_of(meta),
|
|
1268
|
+
(('Content-Disposition', disposition), (SHA_HEADER, meta['sha256'])))
|
|
1269
|
+
return self._send_fd(fd, meta['bytes'], ARTIFACT_TYPES[meta['kind']], ((SHA_HEADER, meta['sha256']),),
|
|
1270
|
+
ARTIFACT_POLICY)
|
|
1271
|
+
finally:
|
|
1272
|
+
os.close(fd)
|
|
1273
|
+
|
|
1274
|
+
def _rendered(self, route, meta, login, source):
|
|
1275
|
+
"""The Markdown document `meta` rendered (None: too large once decoded), from the service's cache by its sha256,
|
|
1276
|
+
else rendered from source()'s bytes in one of RENDER_SLOTS (waiting at most RENDER_WAIT_S, else 503 busy) and
|
|
1277
|
+
kept (F2: a hostile 1 MiB document renders for seconds, and a page could ask for it in a loop)."""
|
|
1278
|
+
service = self.server.service
|
|
1279
|
+
hit, body = service.rendered.get(meta['sha256'])
|
|
1280
|
+
if hit:
|
|
1281
|
+
return body
|
|
1282
|
+
if not service.renders.acquire(timeout=RENDER_WAIT_S):
|
|
1283
|
+
D.audit(route, login, meta['id'], meta['lane'], None, None, 'busy')
|
|
1284
|
+
raise _Reply(503, 'busy', (('Retry-After', str(BUSY_RETRY)),))
|
|
1285
|
+
try:
|
|
1286
|
+
try: # invalid UTF-8 becomes U+FFFD, which can push it past the limit
|
|
1287
|
+
body = mdlite.render(bytes(source()).decode('utf-8', 'replace'))
|
|
1288
|
+
except mdlite.TooLarge:
|
|
1289
|
+
body = None
|
|
1290
|
+
finally:
|
|
1291
|
+
service.renders.release()
|
|
1292
|
+
service.rendered.put(meta['sha256'], body)
|
|
1293
|
+
return body
|
|
1294
|
+
|
|
1295
|
+
def _decrypted(self, route, meta, fd, identity, login):
|
|
1296
|
+
"""The plaintext of an encrypted document (docs_store.decrypt, checked against the receipt), one at a time
|
|
1297
|
+
(_DECRYPTS) and in one of the UPLOAD_SLOTS big buffers it shares with uploads (F8), each waited for at most
|
|
1298
|
+
DECRYPT_WAIT_S, else 503 busy."""
|
|
1299
|
+
service = self.server.service
|
|
1300
|
+
if not _DECRYPTS.acquire(timeout=DECRYPT_WAIT_S):
|
|
1301
|
+
D.audit(route, login, meta['id'], meta['lane'], None, None, 'busy')
|
|
1302
|
+
raise _Reply(503, 'busy', (('Retry-After', str(BUSY_RETRY)),))
|
|
1303
|
+
try:
|
|
1304
|
+
if not service.slots.acquire(timeout=DECRYPT_WAIT_S):
|
|
1305
|
+
D.audit(route, login, meta['id'], meta['lane'], None, None, 'busy')
|
|
1306
|
+
raise _Reply(503, 'busy', (('Retry-After', str(BUSY_RETRY)),))
|
|
1307
|
+
try:
|
|
1308
|
+
return D.decrypt(meta, fd, identity)
|
|
1309
|
+
except D.EncryptError:
|
|
1310
|
+
D.audit(route, login, meta['id'], meta['lane'], None, None, 'encrypt-failed')
|
|
1311
|
+
raise _Reply(500, 'server-error') from None
|
|
1312
|
+
finally:
|
|
1313
|
+
service.slots.release()
|
|
1314
|
+
finally:
|
|
1315
|
+
_DECRYPTS.release()
|
|
1316
|
+
|
|
1317
|
+
def _encrypted(self, route, meta, fd, login):
|
|
1318
|
+
"""An encrypted document (docs_store's `age` switch, meta `enc`; fd its stored `<name>.age`). With an identity
|
|
1319
|
+
(LANES_DOCS_AGE_IDENTITY) and at most MAX_BODY bytes it is decrypted on the fly (_decrypted); raw and artifact
|
|
1320
|
+
then answer as for a plain document, and a view renders it unless the render cache already holds it. Without
|
|
1321
|
+
one, raw and artifact are 409 encrypted (the page names uploads/), and the view shows the facts only."""
|
|
1322
|
+
try:
|
|
1323
|
+
identity = D.age_settings()[1]
|
|
1324
|
+
except D.DocsError:
|
|
1325
|
+
identity = None
|
|
1326
|
+
if route == 'view':
|
|
1327
|
+
body = None
|
|
1328
|
+
if identity and meta['kind'] == 'md' and meta['bytes'] <= mdlite.MAX_BYTES:
|
|
1329
|
+
body = self._rendered(route, meta, login, lambda: self._decrypted(route, meta, fd, identity, login))
|
|
1330
|
+
D.audit(route, login, meta['id'], meta['lane'], meta['bytes'], meta['sha256'], 'ok')
|
|
1331
|
+
return self._page(200, view_page(meta, body, locked=not identity))
|
|
1332
|
+
data = None
|
|
1333
|
+
if identity and meta['bytes'] <= MAX_BODY:
|
|
1334
|
+
data = self._decrypted(route, meta, fd, identity, login)
|
|
1335
|
+
if data is None:
|
|
1336
|
+
D.audit(route, login, meta['id'], meta['lane'], None, None, 'encrypted')
|
|
1337
|
+
raise _Reply(409, 'encrypted')
|
|
1338
|
+
D.audit(route, login, meta['id'], meta['lane'], meta['bytes'], meta['sha256'], 'ok')
|
|
1339
|
+
if route == 'raw':
|
|
1340
|
+
disposition = f'attachment; filename="{download_name(meta["name"])}"'
|
|
1341
|
+
return self._send(200, data, content_type_of(meta),
|
|
1342
|
+
(('Content-Disposition', disposition), (SHA_HEADER, meta['sha256'])))
|
|
1343
|
+
return self._send(200, data, ARTIFACT_TYPES[meta['kind']], ((SHA_HEADER, meta['sha256']),), ARTIFACT_POLICY)
|
|
1344
|
+
|
|
1345
|
+
def _upload(self, service, caller):
|
|
1346
|
+
login = caller.login # the gate refused one from another site already (F2)
|
|
1347
|
+
wait = service.attempt(login)
|
|
1348
|
+
if wait:
|
|
1349
|
+
D.audit('upload', login, None, None, None, None, 'rate-limited')
|
|
1350
|
+
raise _Reply(429, 'rate-limited', (('Retry-After', str(wait)),))
|
|
1351
|
+
if not service.take_upload_slot(login):
|
|
1352
|
+
D.audit('upload', login, None, None, None, None, 'busy')
|
|
1353
|
+
raise _Reply(503, 'busy', (('Retry-After', str(BUSY_RETRY)),))
|
|
1354
|
+
try:
|
|
1355
|
+
return self._upload_in_slot(login)
|
|
1356
|
+
finally:
|
|
1357
|
+
service.give_upload_slot(login)
|
|
1358
|
+
|
|
1359
|
+
def _upload_in_slot(self, login):
|
|
1360
|
+
boundary = form_boundary(self.headers.get('Content-Type'))
|
|
1361
|
+
if boundary is None:
|
|
1362
|
+
D.audit('upload', login, None, None, None, None, 'unsupported-media-type')
|
|
1363
|
+
raise _Reply(415, 'unsupported-media-type')
|
|
1364
|
+
try:
|
|
1365
|
+
data = self._body()
|
|
1366
|
+
except _Reply as reply:
|
|
1367
|
+
D.audit('upload', login, None, None, None, None, reply.error)
|
|
1368
|
+
raise
|
|
1369
|
+
fields = {}
|
|
1370
|
+
reg = registry()
|
|
1371
|
+
try:
|
|
1372
|
+
fields, files = parse_form(data, boundary)
|
|
1373
|
+
if not files:
|
|
1374
|
+
raise FormError('no-file')
|
|
1375
|
+
ws, lane = upload_target(fields, member_workspaces(login), reg)
|
|
1376
|
+
except FormError as error:
|
|
1377
|
+
lane = fields.get('lane')
|
|
1378
|
+
D.audit('upload', login, None, lane if isinstance(lane, str) and L.IDENT.match(lane) else None, None, None,
|
|
1379
|
+
error.error)
|
|
1380
|
+
raise _Reply(503 if error.error == 'registry-unavailable' else 400, error.error) from None
|
|
1381
|
+
workspace = next(entry for entry in reg['workspaces'] if isinstance(entry, dict) and entry.get('id') == ws)
|
|
1382
|
+
try: # where the files land: uploads/ of the workspace (W5)
|
|
1383
|
+
folder, home, parts = U.landing(reg, workspace)
|
|
1384
|
+
uploads = U.open_uploads(home, parts)
|
|
1385
|
+
except (L.LanesError, OSError):
|
|
1386
|
+
D.audit('upload', login, None, lane, None, None, 'uploads-unavailable')
|
|
1387
|
+
raise _Reply(503, 'uploads-unavailable') from None
|
|
1388
|
+
receipts, failure = [], None
|
|
1389
|
+
try:
|
|
1390
|
+
for name, start, end in files:
|
|
1391
|
+
try:
|
|
1392
|
+
meta, landed = U.receive(uploads, D.clean_name(name), _Slice(data, start, end), login=login, ws=ws,
|
|
1393
|
+
lane=lane, source='web')
|
|
1394
|
+
except U.UploadError as error: # intake logged it; nothing of this file landed
|
|
1395
|
+
failure = {'busy': (503, 'busy'), 'encrypt-failed': (500, 'encrypt-failed')}.get(
|
|
1396
|
+
error.word, (400, 'bad-request'))
|
|
1397
|
+
break
|
|
1398
|
+
except (OSError, L.LanesError): # could not land: the store document was removed again
|
|
1399
|
+
D.audit('upload', login, None, lane, None, None, 'uploads-unavailable')
|
|
1400
|
+
failure = (503, 'uploads-unavailable')
|
|
1401
|
+
break
|
|
1402
|
+
receipts.append({'id': meta['id'], 'name': meta['name'], 'sha256': meta['sha256'],
|
|
1403
|
+
'bytes': meta['bytes'], 'url': D.URL.format(meta['id']), 'at': meta['at'],
|
|
1404
|
+
'landed': f'{folder}/uploads/{landed}'})
|
|
1405
|
+
finally:
|
|
1406
|
+
os.close(uploads)
|
|
1407
|
+
status, error = failure if failure else (200, None)
|
|
1408
|
+
headers = (('Retry-After', str(BUSY_RETRY)),) if status == 503 else ()
|
|
1409
|
+
if wants_html(self.headers):
|
|
1410
|
+
return self._page(status, receipts_page(receipts, error), headers)
|
|
1411
|
+
payload = {'receipts': receipts}
|
|
1412
|
+
if error:
|
|
1413
|
+
payload['error'] = error
|
|
1414
|
+
return self._json(status, payload, headers)
|
|
1415
|
+
|
|
1416
|
+
|
|
1417
|
+
class Server(ThreadingHTTPServer):
|
|
1418
|
+
daemon_threads = True
|
|
1419
|
+
request_queue_size = 32
|
|
1420
|
+
|
|
1421
|
+
def __init__(self, address, service):
|
|
1422
|
+
self.service = service
|
|
1423
|
+
super().__init__(address, Handler)
|
|
1424
|
+
|
|
1425
|
+
def server_bind(self): # HTTPServer's getfqdn() lookup is not needed on loopback
|
|
1426
|
+
socketserver.TCPServer.server_bind(self)
|
|
1427
|
+
self.server_name, self.server_port = self.server_address[:2]
|
|
1428
|
+
|
|
1429
|
+
|
|
1430
|
+
# ------------------------------------------------------------------------------------------------ publish pick-up
|
|
1431
|
+
|
|
1432
|
+
PICKUP_INTERVAL = 5 # seconds between two pick-up cycles (lane_core.PUBLISH_LATENCY_S)
|
|
1433
|
+
PUBLISH_LIMIT = MAX_BODY # a published file, at most: the docs limit (lane_core.PUBLISH_MAX)
|
|
1434
|
+
MAX_REQUEST = 16 * 1024 # one request's JSON
|
|
1435
|
+
MAX_PATH = 4096 # a request's `path`
|
|
1436
|
+
MAX_PICKUPS = 16 # requests taken from one container per cycle; the rest wait for the next
|
|
1437
|
+
MAX_RECEIPTS = 16 # receipts kept per lane in a container, the newest
|
|
1438
|
+
PUBLISHES = 30 # requests taken per container in PUBLISH_WINDOW seconds; more are refused
|
|
1439
|
+
PUBLISH_WINDOW = 600
|
|
1440
|
+
PUBLISHED_PUSH_S = 60 # one `published` push per lane in this many seconds, at most
|
|
1441
|
+
PUBLISH_KINDS = ('md', 'html', 'svg', 'file') # lane_core.PUBLISH_KINDS: docs_store.KINDS - upload
|
|
1442
|
+
PUBLISH_DIR = tuple(L.CONTAINER_HOME.strip('/').split('/')) + ('.lz', 'state', 'publish')
|
|
1443
|
+
HOME_PARTS = tuple(L.CONTAINER_HOME.strip('/').split('/'))
|
|
1444
|
+
REQUEST_SUFFIX, RECEIPT_SUFFIX, CLAIM_SUFFIX = '.json', '.receipt.json', '.taking'
|
|
1445
|
+
MAX_ENTRIES = 4096 # directory entries surveyed per container per cycle; more: refused (F1)
|
|
1446
|
+
OVER_CAP_SAY_S = 3600 # an over-cap container is said in the journal once per this many seconds
|
|
1447
|
+
REFUSALS = ('unreadable-request', 'bad-version', 'bad-request', 'bad-kind', 'bad-title', 'too-large', 'unknown-lane',
|
|
1448
|
+
'rate-limited', 'for-not-a-member', 'no-recipient', 'outside-home', 'not-a-file', 'sha256-mismatch',
|
|
1449
|
+
'invalid', 'error') # the words docs.log says; a receipt says receipt_reason()'s
|
|
1450
|
+
NOT_DELIVERED = ('for-not-a-member', 'no-recipient') # their receipt says `not-delivered` (F6)
|
|
1451
|
+
_PLANTED = (errno.EISDIR, errno.ENOTEMPTY, errno.EEXIST) # a rename onto a directory planted at its target
|
|
1452
|
+
_UNCLAIMED = object() # _take: the request could not be claimed; the next one is tried
|
|
1453
|
+
_SHA256 = re.compile(r'[0-9a-f]{64}', re.ASCII)
|
|
1454
|
+
_CHILD_DIR = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | getattr(os, 'O_CLOEXEC', 0)
|
|
1455
|
+
_LEAF = os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK | getattr(os, 'O_CLOEXEC', 0)
|
|
1456
|
+
|
|
1457
|
+
|
|
1458
|
+
class Refused(Exception):
|
|
1459
|
+
"""A publish request refused: `reason` (one of REFUSALS) goes into its receipt, with the host's sha256 and size of
|
|
1460
|
+
the file when they are known."""
|
|
1461
|
+
|
|
1462
|
+
def __init__(self, reason, sha256=None, size=None):
|
|
1463
|
+
super().__init__(reason)
|
|
1464
|
+
self.reason, self.sha256, self.size = reason, sha256, size
|
|
1465
|
+
|
|
1466
|
+
|
|
1467
|
+
def container_roots():
|
|
1468
|
+
"""[(container, root)] for every container containers.json names (`lanes sync`, statusd: L.refresh_containers)
|
|
1469
|
+
whose init pid is alive: root = /proc/<pid>/root. A pid whose root is the host's own `/` is skipped — a container
|
|
1470
|
+
that went away and a host process that got its pid — so the host's home is never read as a container's. And a pid
|
|
1471
|
+
whose /home/coder (opened without following a link) is not the home volume containers.json records for the
|
|
1472
|
+
container (`home`, the host path docker reports; compared by st_dev and st_ino, a mount shows its source's) is
|
|
1473
|
+
skipped too: another container got the pid, and its requests would be taken as this container's lanes' (F10). When
|
|
1474
|
+
this account cannot see the recorded volume, or none is recorded, only the host-root check applies (DEF-1918)."""
|
|
1475
|
+
info = L.read_json(L.STATE / 'containers.json', {}) or {}
|
|
1476
|
+
try:
|
|
1477
|
+
host = os.stat('/')
|
|
1478
|
+
except OSError:
|
|
1479
|
+
host = None
|
|
1480
|
+
found = []
|
|
1481
|
+
for name in sorted(info if isinstance(info, dict) else ()):
|
|
1482
|
+
entry = info[name]
|
|
1483
|
+
pid = entry.get('pid') if isinstance(entry, dict) else None
|
|
1484
|
+
if not L.IDENT.match(name) or not isinstance(pid, int) or isinstance(pid, bool) or pid <= 0:
|
|
1485
|
+
continue
|
|
1486
|
+
root = Path(L.proc_root()) / str(pid) / 'root'
|
|
1487
|
+
try:
|
|
1488
|
+
seen = os.stat(root)
|
|
1489
|
+
except OSError:
|
|
1490
|
+
continue
|
|
1491
|
+
if host is not None and (seen.st_dev, seen.st_ino) == (host.st_dev, host.st_ino):
|
|
1492
|
+
continue
|
|
1493
|
+
if not _home_matches(root, entry.get('home')):
|
|
1494
|
+
continue
|
|
1495
|
+
found.append((name, root))
|
|
1496
|
+
return found
|
|
1497
|
+
|
|
1498
|
+
|
|
1499
|
+
def _home_matches(root, home):
|
|
1500
|
+
"""False only when both the recorded home volume `home` and the container's /home/coder can be seen and they are
|
|
1501
|
+
not the same directory."""
|
|
1502
|
+
if not isinstance(home, str) or not home.startswith('/'):
|
|
1503
|
+
return True
|
|
1504
|
+
try:
|
|
1505
|
+
volume = os.stat(home)
|
|
1506
|
+
fd = L.open_real_dir(root, HOME_PARTS)
|
|
1507
|
+
except (OSError, L.LanesError):
|
|
1508
|
+
return True
|
|
1509
|
+
try:
|
|
1510
|
+
mounted = os.fstat(fd)
|
|
1511
|
+
except OSError:
|
|
1512
|
+
return True
|
|
1513
|
+
finally:
|
|
1514
|
+
os.close(fd)
|
|
1515
|
+
return (volume.st_dev, volume.st_ino) == (mounted.st_dev, mounted.st_ino)
|
|
1516
|
+
|
|
1517
|
+
|
|
1518
|
+
def lane_places(reg):
|
|
1519
|
+
"""{lane id: (workspace id, container)} of every registry lane that runs in a container."""
|
|
1520
|
+
places = {}
|
|
1521
|
+
for workspace in (reg or {}).get('workspaces') or []:
|
|
1522
|
+
if not isinstance(workspace, dict) or not isinstance(workspace.get('id'), str):
|
|
1523
|
+
continue
|
|
1524
|
+
for lane in workspace.get('lanes') or []:
|
|
1525
|
+
if isinstance(lane, dict) and isinstance(lane.get('id'), str) and isinstance(lane.get('container'), str):
|
|
1526
|
+
places.setdefault(lane['id'], (workspace['id'], lane['container']))
|
|
1527
|
+
return places
|
|
1528
|
+
|
|
1529
|
+
|
|
1530
|
+
def read_request(directory, name):
|
|
1531
|
+
"""The checked request in `name` of the open lane directory -> {path, kind, claimed_sha256, bytes, title, for}.
|
|
1532
|
+
Raises Refused (see the module docstring, step 2)."""
|
|
1533
|
+
raw = L.read_regular_at(directory, name, MAX_REQUEST)
|
|
1534
|
+
try:
|
|
1535
|
+
request = json.loads(raw.decode('utf-8')) if raw is not None else None
|
|
1536
|
+
except ValueError:
|
|
1537
|
+
request = None
|
|
1538
|
+
if not isinstance(request, dict):
|
|
1539
|
+
raise Refused('unreadable-request')
|
|
1540
|
+
if request.get('v') != 1:
|
|
1541
|
+
raise Refused('bad-version')
|
|
1542
|
+
path, sha256, size = request.get('path'), request.get('claimed_sha256'), request.get('bytes')
|
|
1543
|
+
title, for_login, mime = request.get('title'), request.get('for'), request.get('mime')
|
|
1544
|
+
if not (isinstance(path, str) and path.startswith('/') and '\x00' not in path and len(path) <= MAX_PATH
|
|
1545
|
+
and isinstance(sha256, str) and _SHA256.fullmatch(sha256)
|
|
1546
|
+
and isinstance(size, int) and not isinstance(size, bool) and size >= 0
|
|
1547
|
+
and all(value is None or isinstance(value, str) for value in (title, for_login, mime))):
|
|
1548
|
+
raise Refused('bad-request')
|
|
1549
|
+
if request.get('kind') not in PUBLISH_KINDS:
|
|
1550
|
+
raise Refused('bad-kind')
|
|
1551
|
+
try:
|
|
1552
|
+
title = D._title(title)
|
|
1553
|
+
except L.LanesError:
|
|
1554
|
+
raise Refused('bad-title') from None
|
|
1555
|
+
if size > PUBLISH_LIMIT:
|
|
1556
|
+
raise Refused('too-large')
|
|
1557
|
+
return {'path': path, 'kind': request['kind'], 'claimed_sha256': sha256, 'bytes': size, 'title': title,
|
|
1558
|
+
'for': for_login or None}
|
|
1559
|
+
|
|
1560
|
+
|
|
1561
|
+
def recipient(lane, ws, for_login, grants, profiles):
|
|
1562
|
+
"""(store login, `for` list) of a document published from `lane` in `ws` (the module docstring, "Who a published
|
|
1563
|
+
document is for"). `grants` is member_grants(), `profiles` users.load_all()'s. Raises Refused."""
|
|
1564
|
+
if for_login is not None:
|
|
1565
|
+
entry = grants.get(for_login.casefold())
|
|
1566
|
+
if entry is None:
|
|
1567
|
+
raise Refused('for-not-a-member')
|
|
1568
|
+
return entry[0], [entry[0]]
|
|
1569
|
+
opened = [(profile['last_opened']['at'], profile['login']) for profile in profiles
|
|
1570
|
+
if isinstance(profile.get('last_opened'), dict) and profile['last_opened'].get('lane') == lane
|
|
1571
|
+
and profile['login'].casefold() in grants]
|
|
1572
|
+
if opened:
|
|
1573
|
+
login = grants[max(opened)[1].casefold()][0]
|
|
1574
|
+
return login, [login]
|
|
1575
|
+
listing = sorted((spelling for spelling, granted in grants.values() if ws in granted), key=str.casefold)
|
|
1576
|
+
if not listing:
|
|
1577
|
+
raise Refused('no-recipient')
|
|
1578
|
+
return listing[0], []
|
|
1579
|
+
|
|
1580
|
+
|
|
1581
|
+
def open_published(root, path):
|
|
1582
|
+
"""(file name, read-only fd) of the file a request names, in the container whose root directory is `root` (the
|
|
1583
|
+
module docstring, step 3). The caller closes the fd. Raises Refused: outside-home, not-a-file, too-large."""
|
|
1584
|
+
try:
|
|
1585
|
+
real = L.container_realpath(root, path)
|
|
1586
|
+
except L.LanesError:
|
|
1587
|
+
raise Refused('not-a-file') from None
|
|
1588
|
+
parts = tuple(part for part in real.split('/') if part)
|
|
1589
|
+
if len(parts) <= len(HOME_PARTS) or parts[:len(HOME_PARTS)] != HOME_PARTS:
|
|
1590
|
+
raise Refused('outside-home')
|
|
1591
|
+
try:
|
|
1592
|
+
directory = L.open_real_dir(root, parts[:-1])
|
|
1593
|
+
except (OSError, L.LanesError):
|
|
1594
|
+
raise Refused('not-a-file') from None
|
|
1595
|
+
try:
|
|
1596
|
+
fd = os.open(parts[-1], _LEAF, dir_fd=directory)
|
|
1597
|
+
except OSError:
|
|
1598
|
+
raise Refused('not-a-file') from None
|
|
1599
|
+
finally:
|
|
1600
|
+
os.close(directory)
|
|
1601
|
+
try:
|
|
1602
|
+
info = os.fstat(fd)
|
|
1603
|
+
if not stat.S_ISREG(info.st_mode):
|
|
1604
|
+
raise Refused('not-a-file')
|
|
1605
|
+
if info.st_size > PUBLISH_LIMIT:
|
|
1606
|
+
raise Refused('too-large', size=info.st_size)
|
|
1607
|
+
except BaseException:
|
|
1608
|
+
os.close(fd)
|
|
1609
|
+
raise
|
|
1610
|
+
return parts[-1], fd
|
|
1611
|
+
|
|
1612
|
+
|
|
1613
|
+
class _Hashing:
|
|
1614
|
+
"""docs_store.intake's reader over the published file: hashes what it hands on and, at the end of the file, raises
|
|
1615
|
+
Refused('sha256-mismatch') for a sha256 other than the claimed one — inside intake, before anything is committed,
|
|
1616
|
+
so intake removes what it wrote."""
|
|
1617
|
+
|
|
1618
|
+
def __init__(self, fd, claimed):
|
|
1619
|
+
self.fd, self.claimed, self.digest, self.size = fd, claimed, hashlib.sha256(), 0
|
|
1620
|
+
|
|
1621
|
+
def read(self, size=-1):
|
|
1622
|
+
chunk = os.read(self.fd, size if isinstance(size, int) and size > 0 else CHUNK)
|
|
1623
|
+
if chunk:
|
|
1624
|
+
self.digest.update(chunk)
|
|
1625
|
+
self.size += len(chunk)
|
|
1626
|
+
elif self.digest.hexdigest() != self.claimed:
|
|
1627
|
+
raise Refused('sha256-mismatch', self.digest.hexdigest(), self.size)
|
|
1628
|
+
return chunk
|
|
1629
|
+
|
|
1630
|
+
|
|
1631
|
+
def receipt_reason(reason):
|
|
1632
|
+
"""The word a receipt says for a refusal: docs.log's, except `not-delivered` for both recipient refusals (F6)."""
|
|
1633
|
+
return 'not-delivered' if reason in NOT_DELIVERED else reason
|
|
1634
|
+
|
|
1635
|
+
|
|
1636
|
+
def receipt_of(request_id, now, meta=None, refusal=None):
|
|
1637
|
+
"""The receipt written back for a request (DM-LANES-PUBLISH-RECEIPT): `meta` of the stored document, or a
|
|
1638
|
+
Refused."""
|
|
1639
|
+
if meta is not None:
|
|
1640
|
+
return {'v': 1, 'id': meta['id'], 'request': request_id, 'sha256': meta['sha256'], 'bytes': meta['bytes'],
|
|
1641
|
+
'url': D.URL.format(meta['id']), 'at': meta['at'], 'outcome': 'ok'}
|
|
1642
|
+
return {'v': 1, 'id': None, 'request': request_id, 'sha256': refusal.sha256, 'bytes': refusal.size, 'url': None,
|
|
1643
|
+
'at': D.stamp(now), 'outcome': 'refused', 'reason': receipt_reason(refusal.reason)}
|
|
1644
|
+
|
|
1645
|
+
|
|
1646
|
+
def prune_receipts(directory, names=None):
|
|
1647
|
+
"""Keep the newest MAX_RECEIPTS receipts of one lane directory (ids sort by their base36 time). `names`: the
|
|
1648
|
+
receipt names the survey saw plus those written since; else the directory is read, at most MAX_ENTRIES entries."""
|
|
1649
|
+
if names is None:
|
|
1650
|
+
names = []
|
|
1651
|
+
try:
|
|
1652
|
+
with os.scandir(directory) as entries:
|
|
1653
|
+
for count, entry in enumerate(entries):
|
|
1654
|
+
if count >= MAX_ENTRIES:
|
|
1655
|
+
return
|
|
1656
|
+
names.append(entry.name)
|
|
1657
|
+
except OSError:
|
|
1658
|
+
return
|
|
1659
|
+
receipts = sorted(name for name in set(names) if name.endswith(RECEIPT_SUFFIX)
|
|
1660
|
+
and D.ID_RE.fullmatch(name[:-len(RECEIPT_SUFFIX)]))
|
|
1661
|
+
for name in receipts[:-MAX_RECEIPTS]:
|
|
1662
|
+
try:
|
|
1663
|
+
os.unlink(name, dir_fd=directory)
|
|
1664
|
+
except OSError:
|
|
1665
|
+
pass
|
|
1666
|
+
|
|
1667
|
+
|
|
1668
|
+
def survey(publish):
|
|
1669
|
+
"""[(lane, pending request ids oldest first, receipt names)] of the open publish directory `publish`, lanes in name
|
|
1670
|
+
order — or None when its directory and its lane directories hold more than MAX_ENTRIES entries between them: the
|
|
1671
|
+
container is refused for this cycle (F1). Read with os.scandir and never past MAX_ENTRIES entries in all, keeping
|
|
1672
|
+
only the names that matter (a lane-id directory; a request or receipt name), so neither time nor memory follows
|
|
1673
|
+
what a container plants."""
|
|
1674
|
+
seen, lanes = 0, []
|
|
1675
|
+
with os.scandir(publish) as entries:
|
|
1676
|
+
for entry in entries:
|
|
1677
|
+
seen += 1
|
|
1678
|
+
if seen > MAX_ENTRIES:
|
|
1679
|
+
return None
|
|
1680
|
+
name = entry.name
|
|
1681
|
+
if not L.IDENT.match(name) or len(name) > users.MAX_LANE_ID:
|
|
1682
|
+
continue
|
|
1683
|
+
try:
|
|
1684
|
+
if entry.is_dir(follow_symlinks=False):
|
|
1685
|
+
lanes.append(name)
|
|
1686
|
+
except OSError:
|
|
1687
|
+
continue
|
|
1688
|
+
found = []
|
|
1689
|
+
for lane in sorted(lanes):
|
|
1690
|
+
try:
|
|
1691
|
+
directory = os.open(lane, _CHILD_DIR, dir_fd=publish)
|
|
1692
|
+
except OSError:
|
|
1693
|
+
continue # a link or a file where a lane directory would be
|
|
1694
|
+
pending, receipts = [], []
|
|
1695
|
+
try:
|
|
1696
|
+
with os.scandir(directory) as entries:
|
|
1697
|
+
for entry in entries:
|
|
1698
|
+
seen += 1
|
|
1699
|
+
if seen > MAX_ENTRIES:
|
|
1700
|
+
return None
|
|
1701
|
+
name = entry.name
|
|
1702
|
+
if name.endswith(RECEIPT_SUFFIX):
|
|
1703
|
+
if D.ID_RE.fullmatch(name[:-len(RECEIPT_SUFFIX)]):
|
|
1704
|
+
receipts.append(name)
|
|
1705
|
+
elif name.endswith(REQUEST_SUFFIX) and D.ID_RE.fullmatch(name[:-len(REQUEST_SUFFIX)]):
|
|
1706
|
+
pending.append(name[:-len(REQUEST_SUFFIX)])
|
|
1707
|
+
except OSError:
|
|
1708
|
+
continue
|
|
1709
|
+
finally:
|
|
1710
|
+
os.close(directory)
|
|
1711
|
+
found.append((lane, sorted(pending), receipts))
|
|
1712
|
+
return found
|
|
1713
|
+
|
|
1714
|
+
|
|
1715
|
+
def _plain(error):
|
|
1716
|
+
"""An OSError without its file names, so one lane's repeated failure is one journal line, not one per request."""
|
|
1717
|
+
return type(error)(error.errno, error.strerror) if isinstance(error, OSError) and error.errno else error
|
|
1718
|
+
|
|
1719
|
+
|
|
1720
|
+
def clear_planted(directory, name):
|
|
1721
|
+
"""Make `name` in the open container directory free for a rename onto it when a directory was planted there: an
|
|
1722
|
+
empty one is removed, any other moved aside under a new dot name (never followed, never read; the host deletes
|
|
1723
|
+
nothing of the container's but its own claims). -> whether the name now holds no directory."""
|
|
1724
|
+
try:
|
|
1725
|
+
info = os.stat(name, dir_fd=directory, follow_symlinks=False)
|
|
1726
|
+
except FileNotFoundError:
|
|
1727
|
+
return True
|
|
1728
|
+
except OSError:
|
|
1729
|
+
return False
|
|
1730
|
+
if not stat.S_ISDIR(info.st_mode):
|
|
1731
|
+
return True
|
|
1732
|
+
try:
|
|
1733
|
+
os.rmdir(name, dir_fd=directory)
|
|
1734
|
+
return True
|
|
1735
|
+
except OSError:
|
|
1736
|
+
pass
|
|
1737
|
+
try:
|
|
1738
|
+
os.rename(name, f'.{name}.{os.urandom(4).hex()}.planted', src_dir_fd=directory, dst_dir_fd=directory)
|
|
1739
|
+
return True
|
|
1740
|
+
except OSError:
|
|
1741
|
+
return False
|
|
1742
|
+
|
|
1743
|
+
|
|
1744
|
+
class Pickup:
|
|
1745
|
+
"""The publish pick-up (the module docstring): one cycle() per PICKUP_INTERVAL in run(), on main()'s thread."""
|
|
1746
|
+
|
|
1747
|
+
def __init__(self, clock=time.time, notify_conf=None, opener=None, tick=None):
|
|
1748
|
+
self.clock, self.notify_conf, self.opener, self.tick = clock, notify_conf, opener, tick
|
|
1749
|
+
self.errors = {} # where -> the last error said there (each distinct one is logged once)
|
|
1750
|
+
self.taken = {} # container -> the times requests were taken (PUBLISHES per PUBLISH_WINDOW)
|
|
1751
|
+
self.pushed = {} # lane -> when its last `published` push was tried
|
|
1752
|
+
self.over_cap = {} # container -> when its over-cap refusal was last said (OVER_CAP_SAY_S)
|
|
1753
|
+
|
|
1754
|
+
def run(self, stop=None):
|
|
1755
|
+
stop = stop or threading.Event()
|
|
1756
|
+
while not stop.is_set():
|
|
1757
|
+
started = time.monotonic()
|
|
1758
|
+
try:
|
|
1759
|
+
self.cycle()
|
|
1760
|
+
except Exception as error: # noqa: BLE001 — one bad cycle never ends the pick-up
|
|
1761
|
+
self.error('cycle', error)
|
|
1762
|
+
if self.tick is not None: # main(): the service's docs.log refusal lines that are due (F4)
|
|
1763
|
+
try:
|
|
1764
|
+
self.tick()
|
|
1765
|
+
except Exception as error: # noqa: BLE001
|
|
1766
|
+
self.error('tick', error)
|
|
1767
|
+
stop.wait(max(0.5, PICKUP_INTERVAL - (time.monotonic() - started)))
|
|
1768
|
+
|
|
1769
|
+
def error(self, where, error):
|
|
1770
|
+
message = f'{type(error).__name__}: {error}'
|
|
1771
|
+
if self.errors.get(where) != message:
|
|
1772
|
+
self.errors[where] = message
|
|
1773
|
+
print(f'lanes-docsd: pick-up {where}: {message}', file=sys.stderr, flush=True)
|
|
1774
|
+
|
|
1775
|
+
def cycle(self):
|
|
1776
|
+
"""One pass over every container -> [(container, lane, request id, outcome)] for the requests taken. Without a
|
|
1777
|
+
registry nothing is taken: no lane can be placed, and a request waits rather than being refused."""
|
|
1778
|
+
reg = registry()
|
|
1779
|
+
if reg is None:
|
|
1780
|
+
return []
|
|
1781
|
+
places, context, published, results = lane_places(reg), {'reg': reg}, [], []
|
|
1782
|
+
for container, root in container_roots():
|
|
1783
|
+
try:
|
|
1784
|
+
results += self._container(container, root, places, context, published)
|
|
1785
|
+
except Exception as error: # noqa: BLE001 — one container's trouble never costs the others
|
|
1786
|
+
self.error(container, error)
|
|
1787
|
+
names = N.lane_names(reg) if published else {}
|
|
1788
|
+
for lane, meta in published:
|
|
1789
|
+
self._push(lane, meta, names)
|
|
1790
|
+
return results
|
|
1791
|
+
|
|
1792
|
+
def _container(self, container, root, places, context, published):
|
|
1793
|
+
try:
|
|
1794
|
+
publish = L.open_real_dir(root, PUBLISH_DIR)
|
|
1795
|
+
except FileNotFoundError:
|
|
1796
|
+
return [] # nothing was ever published from this container
|
|
1797
|
+
except (OSError, L.LanesError) as error:
|
|
1798
|
+
self.error(container, error) # a component that is a link (or not a directory): never followed
|
|
1799
|
+
return []
|
|
1800
|
+
results = []
|
|
1801
|
+
try:
|
|
1802
|
+
lanes = survey(publish)
|
|
1803
|
+
if lanes is None:
|
|
1804
|
+
self._say_over_cap(container)
|
|
1805
|
+
return []
|
|
1806
|
+
for lane, pending, receipts in lanes:
|
|
1807
|
+
if len(results) >= MAX_PICKUPS:
|
|
1808
|
+
break
|
|
1809
|
+
if not pending:
|
|
1810
|
+
continue
|
|
1811
|
+
try:
|
|
1812
|
+
directory = os.open(lane, _CHILD_DIR, dir_fd=publish)
|
|
1813
|
+
except OSError:
|
|
1814
|
+
continue # swapped for a link or a file since the survey
|
|
1815
|
+
try:
|
|
1816
|
+
place = places.get(lane)
|
|
1817
|
+
ws = place[0] if place and place[1] == container else None
|
|
1818
|
+
results += self._lane(container, root, directory, lane, pending, receipts, ws, context,
|
|
1819
|
+
published, MAX_PICKUPS - len(results))
|
|
1820
|
+
finally:
|
|
1821
|
+
os.close(directory)
|
|
1822
|
+
finally:
|
|
1823
|
+
os.close(publish)
|
|
1824
|
+
if results:
|
|
1825
|
+
self.errors.pop(container, None)
|
|
1826
|
+
return results
|
|
1827
|
+
|
|
1828
|
+
def _say_over_cap(self, container):
|
|
1829
|
+
now = self.clock()
|
|
1830
|
+
if now - self.over_cap.get(container, float('-inf')) >= OVER_CAP_SAY_S:
|
|
1831
|
+
self.over_cap[container] = now
|
|
1832
|
+
print(f'lanes-docsd: pick-up {container}: refused: more than {MAX_ENTRIES} entries under '
|
|
1833
|
+
f'{L.CONTAINER_HOME}/.lz/state/publish; nothing is taken from this container until it holds fewer',
|
|
1834
|
+
file=sys.stderr, flush=True)
|
|
1835
|
+
|
|
1836
|
+
def _lane(self, container, root, directory, lane, pending, receipts, ws, context, published, budget):
|
|
1837
|
+
results, unclaimed = [], 0
|
|
1838
|
+
for request_id in pending:
|
|
1839
|
+
if len(results) >= budget or unclaimed >= MAX_PICKUPS:
|
|
1840
|
+
break
|
|
1841
|
+
outcome = self._take(container, root, directory, lane, request_id, ws, context, published)
|
|
1842
|
+
if outcome is None:
|
|
1843
|
+
break # the store is busy: the next cycle tries again
|
|
1844
|
+
if outcome is _UNCLAIMED:
|
|
1845
|
+
unclaimed += 1 # left for the next cycle; the next request is tried (F9)
|
|
1846
|
+
continue
|
|
1847
|
+
results.append((container, lane, request_id, outcome))
|
|
1848
|
+
receipts.append(request_id + RECEIPT_SUFFIX)
|
|
1849
|
+
if results:
|
|
1850
|
+
prune_receipts(directory, receipts)
|
|
1851
|
+
return results
|
|
1852
|
+
|
|
1853
|
+
def _allow(self, container, now):
|
|
1854
|
+
recent = [at for at in self.taken.get(container, ()) if now - at < PUBLISH_WINDOW]
|
|
1855
|
+
allowed = len(recent) < PUBLISHES
|
|
1856
|
+
self.taken[container] = recent + [now] if allowed else recent
|
|
1857
|
+
return allowed
|
|
1858
|
+
|
|
1859
|
+
def _claim(self, directory, request_id, claim, where):
|
|
1860
|
+
"""Rename the request to its claim -> whether it was claimed. A directory planted at the claim's name is cleared
|
|
1861
|
+
(clear_planted) and the rename tried once more (F9); a failure is said once per lane (no file names)."""
|
|
1862
|
+
for attempt in (1, 2):
|
|
1863
|
+
try:
|
|
1864
|
+
os.rename(request_id + REQUEST_SUFFIX, claim, src_dir_fd=directory, dst_dir_fd=directory)
|
|
1865
|
+
return True
|
|
1866
|
+
except OSError as error:
|
|
1867
|
+
if attempt == 1 and error.errno in _PLANTED and clear_planted(directory, claim):
|
|
1868
|
+
continue
|
|
1869
|
+
self.error(where, _plain(error))
|
|
1870
|
+
return False
|
|
1871
|
+
return False
|
|
1872
|
+
|
|
1873
|
+
def _write_receipt(self, directory, name, data, where):
|
|
1874
|
+
"""The receipt, by L.atomic_write_at (a rename onto `name`); a directory planted there is cleared first and the
|
|
1875
|
+
write tried once more (F9: before, the rename failed and the container never got its answer)."""
|
|
1876
|
+
for attempt in (1, 2):
|
|
1877
|
+
try:
|
|
1878
|
+
L.atomic_write_at(directory, name, data, mode=0o644)
|
|
1879
|
+
return True
|
|
1880
|
+
except OSError as error:
|
|
1881
|
+
if attempt == 1 and error.errno in _PLANTED and clear_planted(directory, name):
|
|
1882
|
+
continue
|
|
1883
|
+
self.error(where, _plain(error))
|
|
1884
|
+
except L.LanesError as error:
|
|
1885
|
+
self.error(where, error)
|
|
1886
|
+
return False
|
|
1887
|
+
return False
|
|
1888
|
+
|
|
1889
|
+
def _take(self, container, root, directory, lane, request_id, ws, context, published):
|
|
1890
|
+
"""One request -> 'ok' or its refusal reason (docs.log's word); _UNCLAIMED when the claim could not be made
|
|
1891
|
+
(the request is left; the caller tries the next one); None when the store was busy (the claim is undone and
|
|
1892
|
+
the request waits for the next cycle)."""
|
|
1893
|
+
claim = f'.{request_id}{CLAIM_SUFFIX}'
|
|
1894
|
+
if not self._claim(directory, request_id, claim, f'{container}/{lane}'):
|
|
1895
|
+
return _UNCLAIMED
|
|
1896
|
+
now, login, meta = self.clock(), None, None
|
|
1897
|
+
try:
|
|
1898
|
+
request = read_request(directory, claim)
|
|
1899
|
+
if ws is None:
|
|
1900
|
+
raise Refused('unknown-lane')
|
|
1901
|
+
if not self._allow(container, now):
|
|
1902
|
+
raise Refused('rate-limited')
|
|
1903
|
+
if 'grants' not in context:
|
|
1904
|
+
context['grants'], context['profiles'] = member_grants(context['reg']), users.load_all()[0]
|
|
1905
|
+
login, fors = recipient(lane, ws, request['for'], context['grants'], context['profiles'])
|
|
1906
|
+
name, fd = open_published(root, request['path'])
|
|
1907
|
+
try:
|
|
1908
|
+
meta = D.intake(login, ws, lane, name, _Hashing(fd, request['claimed_sha256']), limit=PUBLISH_LIMIT,
|
|
1909
|
+
source='publish', kind=request['kind'], title=request['title'], for_logins=fors,
|
|
1910
|
+
by=lane)
|
|
1911
|
+
finally:
|
|
1912
|
+
os.close(fd)
|
|
1913
|
+
except D.Busy:
|
|
1914
|
+
try: # undo the claim: the request waits for the next cycle
|
|
1915
|
+
os.rename(claim, request_id + REQUEST_SUFFIX, src_dir_fd=directory, dst_dir_fd=directory)
|
|
1916
|
+
except OSError as error:
|
|
1917
|
+
self.error(f'{container}/{lane}', error)
|
|
1918
|
+
return None
|
|
1919
|
+
except Refused as error:
|
|
1920
|
+
refusal = error
|
|
1921
|
+
except D.TooLarge as error:
|
|
1922
|
+
refusal = Refused('too-large', size=error.received)
|
|
1923
|
+
except D.DocsError:
|
|
1924
|
+
refusal = Refused('invalid')
|
|
1925
|
+
except Exception as error: # noqa: BLE001 — claimed: the request is answered, never dropped silently
|
|
1926
|
+
self.error(f'{container}/{lane}', error)
|
|
1927
|
+
refusal = Refused('error')
|
|
1928
|
+
else:
|
|
1929
|
+
refusal = None
|
|
1930
|
+
if refusal is not None:
|
|
1931
|
+
receipt, refusal_word = receipt_of(request_id, now, refusal=refusal), refusal.reason
|
|
1932
|
+
else:
|
|
1933
|
+
receipt = receipt_of(request_id, now, meta=meta)
|
|
1934
|
+
published.append((lane, meta))
|
|
1935
|
+
outcome = 'ok' if meta is not None else refusal_word
|
|
1936
|
+
D.audit('publish', login, receipt['id'], lane, receipt['bytes'], receipt['sha256'], outcome, now)
|
|
1937
|
+
self._write_receipt(directory, request_id + RECEIPT_SUFFIX,
|
|
1938
|
+
(json.dumps(receipt, indent=1, ensure_ascii=False) + '\n').encode('utf-8'),
|
|
1939
|
+
f'{container}/{lane}')
|
|
1940
|
+
try:
|
|
1941
|
+
os.unlink(claim, dir_fd=directory)
|
|
1942
|
+
except IsADirectoryError:
|
|
1943
|
+
try:
|
|
1944
|
+
os.rmdir(claim, dir_fd=directory)
|
|
1945
|
+
except OSError:
|
|
1946
|
+
pass # a non-empty directory planted as a request: left under its dot name
|
|
1947
|
+
except OSError as error:
|
|
1948
|
+
self.error(f'{container}/{lane}', error)
|
|
1949
|
+
return outcome
|
|
1950
|
+
|
|
1951
|
+
def _push(self, lane, meta, names):
|
|
1952
|
+
now = self.clock()
|
|
1953
|
+
if now - self.pushed.get(lane, float('-inf')) < PUBLISHED_PUSH_S:
|
|
1954
|
+
return None
|
|
1955
|
+
self.pushed[lane] = now
|
|
1956
|
+
try:
|
|
1957
|
+
return N.notify_published(lane, meta, names, conf_path=self.notify_conf, state=L.STATE, now=now,
|
|
1958
|
+
opener=self.opener)
|
|
1959
|
+
except Exception as error: # noqa: BLE001 — a push is best effort
|
|
1960
|
+
self.error('notify', error)
|
|
1961
|
+
return None
|
|
1962
|
+
|
|
1963
|
+
|
|
1964
|
+
def pin_state():
|
|
1965
|
+
mode, problem = E.read_mode()
|
|
1966
|
+
if mode == 'request':
|
|
1967
|
+
return f'pinned request ({E.mode_path()})'
|
|
1968
|
+
return (f'NOT pinned ({problem or ("pinned " + mode if mode else "auto")}): every request is refused until the '
|
|
1969
|
+
'Serve header check passes and `lanes enrol mode request` pins it')
|
|
1970
|
+
|
|
1971
|
+
|
|
1972
|
+
def main(argv=None):
|
|
1973
|
+
parser = argparse.ArgumentParser(prog='lanes-docsd', description='Lanes docs service (loopback only).')
|
|
1974
|
+
parser.add_argument('--port', type=int, default=None, help=f'default $LANES_DOCS_PORT or {DEFAULT_PORT}')
|
|
1975
|
+
parser.add_argument('--bind', default=DEFAULT_BIND, help=f'a loopback address (default {DEFAULT_BIND})')
|
|
1976
|
+
args = parser.parse_args(argv)
|
|
1977
|
+
port = args.port
|
|
1978
|
+
if port is None:
|
|
1979
|
+
try:
|
|
1980
|
+
port = int(os.environ.get('LANES_DOCS_PORT') or DEFAULT_PORT)
|
|
1981
|
+
except ValueError:
|
|
1982
|
+
parser.error('LANES_DOCS_PORT is not a port number')
|
|
1983
|
+
if not 0 <= port <= 65535:
|
|
1984
|
+
parser.error(f'{port} is not a port number')
|
|
1985
|
+
try:
|
|
1986
|
+
loopback = ipaddress.ip_address(args.bind).is_loopback
|
|
1987
|
+
except ValueError:
|
|
1988
|
+
loopback = False
|
|
1989
|
+
if not loopback:
|
|
1990
|
+
parser.error('--bind takes a loopback address: the service trusts the identity headers of its proxy chain')
|
|
1991
|
+
service = Service.from_environ(os.environ)
|
|
1992
|
+
server = Server((args.bind, port), service)
|
|
1993
|
+
L.record_tree('docsd') # W9 U0: the tree this service runs from (`lanes version`, doctor)
|
|
1994
|
+
# The publish pick-up (plan W10 W4): its own 5 s cycle on a daemon thread, beside the HTTP server; each cycle also
|
|
1995
|
+
# writes the service's due `denied-repeat` lines (F4).
|
|
1996
|
+
threading.Thread(target=Pickup(tick=service.flush_denied).run, name='lanes-docsd-pickup', daemon=True).start()
|
|
1997
|
+
print(f'lanes-docsd {version()}: listening on {args.bind}:{server.server_address[1]}; {pin_state()}; '
|
|
1998
|
+
f'domains {",".join(service.domains)}; store {D.root()}; publish pick-up every {PICKUP_INTERVAL} s',
|
|
1999
|
+
file=sys.stderr, flush=True)
|
|
2000
|
+
try:
|
|
2001
|
+
server.serve_forever()
|
|
2002
|
+
except KeyboardInterrupt:
|
|
2003
|
+
pass
|
|
2004
|
+
finally:
|
|
2005
|
+
server.server_close()
|
|
2006
|
+
return 0
|
|
2007
|
+
|
|
2008
|
+
|
|
2009
|
+
if __name__ == '__main__':
|
|
2010
|
+
sys.exit(main())
|