loki-mode 7.78.0 → 7.80.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -2,7 +2,7 @@
2
2
 
3
3
  The flagship product of [Autonomi](https://www.autonomi.dev/). Loki Mode is a spec-driven autonomous builder with a built-in trust layer that takes any spec to a deployed product and verifies completion with evidence (quality gates plus a completion council), not just a "done" claim. Complete installation instructions for all platforms and use cases.
4
4
 
5
- **Version:** v7.78.0
5
+ **Version:** v7.80.0
6
6
 
7
7
  ---
8
8
 
@@ -395,7 +395,7 @@ provider works inside the container. Provide auth with your Anthropic API key:
395
395
  # Run Loki Mode in Docker (Claude provider, API-key auth)
396
396
  docker run --rm -e ANTHROPIC_API_KEY="$ANTHROPIC_API_KEY" \
397
397
  -v $(pwd):/workspace -w /workspace \
398
- asklokesh/loki-mode:7.78.0 start ./my-spec.md
398
+ asklokesh/loki-mode:7.80.0 start ./my-spec.md
399
399
  ```
400
400
 
401
401
  ##### docker compose + .env (no host install)
@@ -1,5 +1,5 @@
1
1
  // @bun
2
- var QQ=Object.defineProperty;var ZQ=($)=>$;function zQ($,Q){this[$]=ZQ.bind(null,Q)}var b=($,Q)=>{for(var Z in Q)QQ($,Z,{get:Q[Z],enumerable:!0,configurable:!0,set:zQ.bind(Q,Z)})};var L=($,Q)=>()=>($&&(Q=$($=0)),Q);var q$=import.meta.require;var h1={};b(h1,{lokiDir:()=>P,homeLokiDir:()=>i$,findRepoRootForVersion:()=>t$,REPO_ROOT:()=>g});import{resolve as a,dirname as r$}from"path";import{fileURLToPath as XQ}from"url";import{existsSync as R$}from"fs";import{homedir as KQ}from"os";function qQ(){let $=b1;for(let Q=0;Q<6;Q++){if(R$(a($,"VERSION"))&&R$(a($,"autonomy/run.sh")))return $;let Z=r$($);if(Z===$)break;$=Z}return a(b1,"..","..","..")}function t$($){let Q=$;for(let Z=0;Z<6;Z++){if(R$(a(Q,"VERSION"))&&R$(a(Q,"autonomy/run.sh")))return Q;let z=r$(Q);if(z===Q)break;Q=z}return a($,"..","..","..")}function P(){return process.env.LOKI_DIR??a(process.cwd(),".loki")}function i$(){return a(KQ(),".loki")}var b1,g;var C=L(()=>{b1=r$(XQ(import.meta.url));g=qQ()});import{readFileSync as VQ}from"fs";import{resolve as JQ,dirname as UQ}from"path";import{fileURLToPath as WQ}from"url";function E$(){if(Q$!==null)return Q$;let $="7.78.0";if(typeof $==="string"&&$.length>0)return Q$=$,Q$;try{let Q=UQ(WQ(import.meta.url)),Z=t$(Q);Q$=VQ(JQ(Z,"VERSION"),"utf-8").trim()}catch{Q$="unknown"}return Q$}var Q$=null;var e$=L(()=>{C()});var g1={};b(g1,{runOrThrow:()=>HQ,run:()=>F,readStreamCapped:()=>m1,commandVersion:()=>BQ,commandExists:()=>f,ShellError:()=>$1,MAX_STDOUT_BYTES:()=>v1});async function m1($,Q=v1){let Z=$.getReader(),z=new TextDecoder,X="",q=0;try{while(q<Q){let{done:K,value:U}=await Z.read();if(K)break;if(!U)continue;if(q+=U.byteLength,q>Q){let J=U.byteLength-(q-Q);X+=z.decode(U.subarray(0,J),{stream:!0});break}X+=z.decode(U,{stream:!0})}X+=z.decode()}finally{try{await Z.cancel()}catch{}Z.releaseLock()}return X}async function F($,Q={}){let Z=Bun.spawn({cmd:[...$],stdout:"pipe",stderr:"pipe",env:Q.env?{...process.env,...Q.env}:process.env,cwd:Q.cwd}),z,X;if(Q.timeoutMs&&Q.timeoutMs>0)z=setTimeout(()=>{try{Z.kill("SIGTERM")}catch{}X=setTimeout(()=>{try{Z.kill("SIGKILL")}catch{}},2000)},Q.timeoutMs);try{let[q,K,U]=await Promise.all([m1(Z.stdout),new Response(Z.stderr).text(),Z.exited]);return{stdout:q,stderr:K,exitCode:U}}finally{if(z)clearTimeout(z);if(X)clearTimeout(X)}}async function HQ($,Q={}){let Z=await F($,Q);if(Z.exitCode!==0)throw new $1(`command failed (${Z.exitCode}): ${$.join(" ")}`,Z.exitCode,Z.stdout,Z.stderr);return Z}async function f($){let Q=GQ($),Z=await F(["sh","-c",`command -v ${Q}`],{timeoutMs:5000});if(Z.exitCode===0)return Z.stdout.trim()||null;return null}function GQ($){if(!/^[A-Za-z0-9._/-]+$/.test($))throw Error(`refused to shell-escape suspect token: ${$}`);return $}async function BQ($,Q="--version"){if(!await f($))return null;let z=await F([$,Q],{timeoutMs:5000});if(z.exitCode!==0)return null;return((z.stdout||z.stderr).split(/\r?\n/)[0]?.trim()??"")||null}var v1=16777216,$1;var d=L(()=>{$1=class $1 extends Error{message;exitCode;stdout;stderr;constructor($,Q,Z,z){super($);this.message=$;this.exitCode=Q;this.stdout=Z;this.stderr=z;this.name="ShellError"}}});function s($){return YQ?"":$}var YQ,O,S,_,_Z,I,k,h,V;var c=L(()=>{YQ=(process.env.NO_COLOR??"").length>0;O=s("\x1B[0;31m"),S=s("\x1B[0;32m"),_=s("\x1B[1;33m"),_Z=s("\x1B[0;34m"),I=s("\x1B[0;36m"),k=s("\x1B[1m"),h=s("\x1B[2m"),V=s("\x1B[0m")});import{existsSync as jQ}from"fs";async function Z$(){if(Y$!==void 0)return Y$;let $="/opt/homebrew/bin/python3.12";if(jQ($))return Y$=$,$;let Q=await f("python3.12");if(Q)return Y$=Q,Q;let Z=await f("python3");return Y$=Z,Z}async function z$($,Q={}){let Z=await Z$();if(!Z)return{stdout:"",stderr:"python3 not found",exitCode:127};return F([Z,"-c",$],Q)}var Y$;var V$=L(()=>{d()});var X0={};b(X0,{runStatus:()=>oQ});import{existsSync as y,readFileSync as U$,readdirSync as r1,statSync as t1}from"fs";import{resolve as D,basename as vQ}from"path";import{homedir as mQ}from"os";function i1($){let Q=Math.trunc($);if(Q>=1e6)return`${(Math.trunc(Q/1e6*10)/10).toFixed(1)}M`;if(Q>=1000)return`${(Math.trunc(Q/1000*10)/10).toFixed(1)}K`;return String(Q)}function e1($,Q,Z){if(Q===0)return null;let z=Math.trunc($*100/Q),X=Math.trunc($*N$/Q);if(X>N$)X=N$;let q=N$-X,K=S;if(z>=80)K=O;else if(z>=50)K=_;let U="=".repeat(Math.max(0,X))+" ".repeat(Math.max(0,q)),J=i1($),W=i1(Q);return` ${k}${Z}${V} ${K}[${U}]${V} ${z}% (${J} / ${W})`}async function fQ(){if(await f("jq"))return!0;return process.stdout.write(`${O}Error: jq is required but not installed.${V}
2
+ var QQ=Object.defineProperty;var ZQ=($)=>$;function zQ($,Q){this[$]=ZQ.bind(null,Q)}var b=($,Q)=>{for(var Z in Q)QQ($,Z,{get:Q[Z],enumerable:!0,configurable:!0,set:zQ.bind(Q,Z)})};var L=($,Q)=>()=>($&&(Q=$($=0)),Q);var q$=import.meta.require;var h1={};b(h1,{lokiDir:()=>P,homeLokiDir:()=>i$,findRepoRootForVersion:()=>t$,REPO_ROOT:()=>g});import{resolve as a,dirname as r$}from"path";import{fileURLToPath as XQ}from"url";import{existsSync as R$}from"fs";import{homedir as KQ}from"os";function qQ(){let $=b1;for(let Q=0;Q<6;Q++){if(R$(a($,"VERSION"))&&R$(a($,"autonomy/run.sh")))return $;let Z=r$($);if(Z===$)break;$=Z}return a(b1,"..","..","..")}function t$($){let Q=$;for(let Z=0;Z<6;Z++){if(R$(a(Q,"VERSION"))&&R$(a(Q,"autonomy/run.sh")))return Q;let z=r$(Q);if(z===Q)break;Q=z}return a($,"..","..","..")}function P(){return process.env.LOKI_DIR??a(process.cwd(),".loki")}function i$(){return a(KQ(),".loki")}var b1,g;var C=L(()=>{b1=r$(XQ(import.meta.url));g=qQ()});import{readFileSync as VQ}from"fs";import{resolve as JQ,dirname as UQ}from"path";import{fileURLToPath as WQ}from"url";function E$(){if(Q$!==null)return Q$;let $="7.80.0";if(typeof $==="string"&&$.length>0)return Q$=$,Q$;try{let Q=UQ(WQ(import.meta.url)),Z=t$(Q);Q$=VQ(JQ(Z,"VERSION"),"utf-8").trim()}catch{Q$="unknown"}return Q$}var Q$=null;var e$=L(()=>{C()});var g1={};b(g1,{runOrThrow:()=>HQ,run:()=>F,readStreamCapped:()=>m1,commandVersion:()=>BQ,commandExists:()=>f,ShellError:()=>$1,MAX_STDOUT_BYTES:()=>v1});async function m1($,Q=v1){let Z=$.getReader(),z=new TextDecoder,X="",q=0;try{while(q<Q){let{done:K,value:U}=await Z.read();if(K)break;if(!U)continue;if(q+=U.byteLength,q>Q){let J=U.byteLength-(q-Q);X+=z.decode(U.subarray(0,J),{stream:!0});break}X+=z.decode(U,{stream:!0})}X+=z.decode()}finally{try{await Z.cancel()}catch{}Z.releaseLock()}return X}async function F($,Q={}){let Z=Bun.spawn({cmd:[...$],stdout:"pipe",stderr:"pipe",env:Q.env?{...process.env,...Q.env}:process.env,cwd:Q.cwd}),z,X;if(Q.timeoutMs&&Q.timeoutMs>0)z=setTimeout(()=>{try{Z.kill("SIGTERM")}catch{}X=setTimeout(()=>{try{Z.kill("SIGKILL")}catch{}},2000)},Q.timeoutMs);try{let[q,K,U]=await Promise.all([m1(Z.stdout),new Response(Z.stderr).text(),Z.exited]);return{stdout:q,stderr:K,exitCode:U}}finally{if(z)clearTimeout(z);if(X)clearTimeout(X)}}async function HQ($,Q={}){let Z=await F($,Q);if(Z.exitCode!==0)throw new $1(`command failed (${Z.exitCode}): ${$.join(" ")}`,Z.exitCode,Z.stdout,Z.stderr);return Z}async function f($){let Q=GQ($),Z=await F(["sh","-c",`command -v ${Q}`],{timeoutMs:5000});if(Z.exitCode===0)return Z.stdout.trim()||null;return null}function GQ($){if(!/^[A-Za-z0-9._/-]+$/.test($))throw Error(`refused to shell-escape suspect token: ${$}`);return $}async function BQ($,Q="--version"){if(!await f($))return null;let z=await F([$,Q],{timeoutMs:5000});if(z.exitCode!==0)return null;return((z.stdout||z.stderr).split(/\r?\n/)[0]?.trim()??"")||null}var v1=16777216,$1;var d=L(()=>{$1=class $1 extends Error{message;exitCode;stdout;stderr;constructor($,Q,Z,z){super($);this.message=$;this.exitCode=Q;this.stdout=Z;this.stderr=z;this.name="ShellError"}}});function s($){return YQ?"":$}var YQ,O,S,_,_Z,I,k,h,V;var c=L(()=>{YQ=(process.env.NO_COLOR??"").length>0;O=s("\x1B[0;31m"),S=s("\x1B[0;32m"),_=s("\x1B[1;33m"),_Z=s("\x1B[0;34m"),I=s("\x1B[0;36m"),k=s("\x1B[1m"),h=s("\x1B[2m"),V=s("\x1B[0m")});import{existsSync as jQ}from"fs";async function Z$(){if(Y$!==void 0)return Y$;let $="/opt/homebrew/bin/python3.12";if(jQ($))return Y$=$,$;let Q=await f("python3.12");if(Q)return Y$=Q,Q;let Z=await f("python3");return Y$=Z,Z}async function z$($,Q={}){let Z=await Z$();if(!Z)return{stdout:"",stderr:"python3 not found",exitCode:127};return F([Z,"-c",$],Q)}var Y$;var V$=L(()=>{d()});var X0={};b(X0,{runStatus:()=>oQ});import{existsSync as y,readFileSync as U$,readdirSync as r1,statSync as t1}from"fs";import{resolve as D,basename as vQ}from"path";import{homedir as mQ}from"os";function i1($){let Q=Math.trunc($);if(Q>=1e6)return`${(Math.trunc(Q/1e6*10)/10).toFixed(1)}M`;if(Q>=1000)return`${(Math.trunc(Q/1000*10)/10).toFixed(1)}K`;return String(Q)}function e1($,Q,Z){if(Q===0)return null;let z=Math.trunc($*100/Q),X=Math.trunc($*N$/Q);if(X>N$)X=N$;let q=N$-X,K=S;if(z>=80)K=O;else if(z>=50)K=_;let U="=".repeat(Math.max(0,X))+" ".repeat(Math.max(0,q)),J=i1($),W=i1(Q);return` ${k}${Z}${V} ${K}[${U}]${V} ${z}% (${J} / ${W})`}async function fQ(){if(await f("jq"))return!0;return process.stdout.write(`${O}Error: jq is required but not installed.${V}
3
3
  `),process.stdout.write(`Install with:
4
4
  `),process.stdout.write(` brew install jq (macOS)
5
5
  `),process.stdout.write(` apt install jq (Debian/Ubuntu)
@@ -793,4 +793,4 @@ Set LOKI_LEGACY_BASH=1 to force the bash CLI for every command.
793
793
  `),2}default:return process.stderr.write(`Unknown command: ${Q}
794
794
  `),process.stderr.write($Q),2}}s1();process.on("SIGINT",()=>process.exit(130));process.on("SIGTERM",()=>process.exit(143));var qZ=await KZ(Bun.argv.slice(2));process.exit(qZ);
795
795
 
796
- //# debugId=753B9888BD0ED74B64756E2164756E21
796
+ //# debugId=5DBFF3AC2E319A5664756E2164756E21
@@ -0,0 +1,65 @@
1
+ """
2
+ LokiStore: pluggable, local-first storage abstraction for Loki Mode.
3
+
4
+ This package is the consolidation spine for durable artifacts (state,
5
+ checkpoints, memory blobs, healing artifacts, etc.). Features should bind to
6
+ this abstraction instead of hardcoding `.loki/` paths or embedding their own
7
+ cloud SDK clients.
8
+
9
+ Design goals
10
+ ------------
11
+ - Local-first by default. With nothing configured, LocalStore writes to the
12
+ project `.loki/` directory with byte-identical behavior to today's direct
13
+ file writes (atomic temp+rename, fcntl advisory locking, path-traversal
14
+ guard). Zero new dependencies, zero behavior change for local users.
15
+ - Optional cloud backends (S3, GCS, Azure Blob) behind lazy imports. The
16
+ cloud SDK is imported ONLY inside the backend's __init__, so a missing
17
+ dependency raises a clear, actionable error ONLY when that backend is
18
+ explicitly selected -- never at import time, never for local users.
19
+ - One interface (put/get/get_to/exists/list/delete) so callers are backend
20
+ agnostic.
21
+
22
+ Quick start
23
+ -----------
24
+ from lokistore import get_store
25
+
26
+ store = get_store() # local by default, honors LOKI_DIR/TARGET_DIR
27
+ store.put("state/checkpoints/cp-1/metadata.json", b"{...}")
28
+ data = store.get("state/checkpoints/cp-1/metadata.json")
29
+ for key in store.list("state/checkpoints/"):
30
+ ...
31
+
32
+ Configuration (env or config dict)
33
+ ----------------------------------
34
+ - LOKI_STORAGE_BACKEND : local | s3 | gcs | azure-blob (default: local)
35
+ - LOKI_STORAGE_BUCKET : bucket / container name (cloud backends)
36
+ - LOKI_STORAGE_PREFIX : key prefix within the bucket (optional)
37
+ - LOKI_STORAGE_REGION : region (s3, optional)
38
+
39
+ The metadata backend (sqlite default, postgres later) is selected separately;
40
+ see get_metadata_backend() / METADATA notes below. For this release the blob
41
+ backends are the core deliverable and the metadata selector is a thin,
42
+ documented stub that defaults to the existing sqlite path.
43
+ """
44
+
45
+ from .base import LokiStore, StoreError, BackendNotAvailableError
46
+ from .local import LocalStore
47
+ from .factory import (
48
+ get_store,
49
+ build_store,
50
+ get_metadata_backend,
51
+ resolve_local_base,
52
+ )
53
+
54
+ __all__ = [
55
+ "LokiStore",
56
+ "StoreError",
57
+ "BackendNotAvailableError",
58
+ "LocalStore",
59
+ "get_store",
60
+ "build_store",
61
+ "get_metadata_backend",
62
+ "resolve_local_base",
63
+ ]
64
+
65
+ __version__ = "1.0.0"
@@ -0,0 +1,172 @@
1
+ """
2
+ LokiStore interface and shared helpers.
3
+
4
+ Defines the abstract blob-store contract that every backend implements, plus
5
+ errors and a key-normalization helper shared across backends.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import os
11
+ from abc import ABC, abstractmethod
12
+ from typing import List, Union
13
+
14
+
15
+ class StoreError(Exception):
16
+ """Base class for LokiStore errors."""
17
+
18
+
19
+ class BackendNotAvailableError(StoreError):
20
+ """
21
+ Raised when a cloud backend is explicitly selected but its SDK (or
22
+ required configuration) is not available. The message names the missing
23
+ package and how to install it so the failure is actionable.
24
+ """
25
+
26
+
27
+ def normalize_key(key: str) -> str:
28
+ """
29
+ Normalize a store key to a clean POSIX-style relative path.
30
+
31
+ Keys are relative paths like "state/checkpoints/cp-1/metadata.json".
32
+ This rejects absolute paths and any key that would traverse outside the
33
+ store root (a leading or embedded ".." segment). The same guard is used
34
+ by every backend so traversal is rejected uniformly, whether the bytes
35
+ land on a local filesystem or in an object-store key namespace.
36
+
37
+ Args:
38
+ key: A POSIX-style relative path.
39
+
40
+ Returns:
41
+ The normalized key (forward slashes, no leading slash, no "."/".."
42
+ segments collapsed away).
43
+
44
+ Raises:
45
+ ValueError: If the key is empty, absolute, or escapes the root.
46
+ """
47
+ if not isinstance(key, str) or not key:
48
+ raise ValueError("store key must be a non-empty string")
49
+
50
+ # Treat backslashes as separators too, so a Windows-style key cannot
51
+ # smuggle a traversal past the split() check below.
52
+ candidate = key.replace("\\", "/")
53
+
54
+ if candidate.startswith("/"):
55
+ raise ValueError(f"absolute keys are not allowed: {key!r}")
56
+
57
+ parts: List[str] = []
58
+ for segment in candidate.split("/"):
59
+ if segment == "" or segment == ".":
60
+ # Collapse empty (e.g. "a//b") and current-dir segments.
61
+ continue
62
+ if segment == "..":
63
+ raise ValueError(f"path traversal is not allowed in key: {key!r}")
64
+ parts.append(segment)
65
+
66
+ if not parts:
67
+ raise ValueError(f"key resolves to an empty path: {key!r}")
68
+
69
+ return "/".join(parts)
70
+
71
+
72
+ def read_source_bytes(data: Union[bytes, bytearray, str, os.PathLike]) -> bytes:
73
+ """
74
+ Coerce a put() source argument into bytes.
75
+
76
+ Accepts raw bytes (returned as-is) or a path-like pointing at a file whose
77
+ contents are read in binary. A plain str is treated as a FILESYSTEM PATH,
78
+ not as text, to match the put(key, bytes_or_path) contract; callers that
79
+ want to store a string must encode it themselves.
80
+
81
+ Args:
82
+ data: bytes/bytearray, or a path-like to an existing file.
83
+
84
+ Returns:
85
+ The bytes to store.
86
+
87
+ Raises:
88
+ FileNotFoundError: If a path is given and the file does not exist.
89
+ TypeError: If the argument is neither bytes nor a path-like.
90
+ """
91
+ if isinstance(data, (bytes, bytearray)):
92
+ return bytes(data)
93
+ if isinstance(data, (str, os.PathLike)):
94
+ with open(data, "rb") as f:
95
+ return f.read()
96
+ raise TypeError(
97
+ "put() source must be bytes or a path-like to a file, "
98
+ f"got {type(data).__name__}"
99
+ )
100
+
101
+
102
+ class LokiStore(ABC):
103
+ """
104
+ Abstract pluggable blob store.
105
+
106
+ All keys are POSIX-style relative paths (see normalize_key). Implementations
107
+ must apply normalize_key to every incoming key so traversal is rejected and
108
+ behavior is uniform across backends.
109
+
110
+ The contract is intentionally small (no streaming, no multipart, no
111
+ credential framework) so backends stay thin. Larger payload handling can be
112
+ layered on later without changing this interface.
113
+ """
114
+
115
+ @abstractmethod
116
+ def put(self, key: str, data: Union[bytes, bytearray, str, os.PathLike]) -> None:
117
+ """
118
+ Store bytes under key. The source is either raw bytes or a path-like
119
+ to a file whose contents are stored. Overwrites any existing value
120
+ atomically where the backend supports it.
121
+ """
122
+
123
+ @abstractmethod
124
+ def get(self, key: str) -> bytes:
125
+ """
126
+ Return the bytes stored under key.
127
+
128
+ Raises:
129
+ FileNotFoundError: If the key does not exist.
130
+ """
131
+
132
+ @abstractmethod
133
+ def get_to(self, key: str, dest_path: Union[str, os.PathLike]) -> None:
134
+ """
135
+ Write the bytes stored under key to dest_path (a local filesystem
136
+ path). Parent directories are created as needed.
137
+
138
+ Raises:
139
+ FileNotFoundError: If the key does not exist.
140
+ """
141
+
142
+ @abstractmethod
143
+ def exists(self, key: str) -> bool:
144
+ """Return True if key exists in the store."""
145
+
146
+ @abstractmethod
147
+ def list(self, prefix: str = "") -> List[str]:
148
+ """
149
+ List keys under prefix (a POSIX-style relative PATH prefix). An empty
150
+ prefix lists every key. Returns normalized keys, sorted, relative to
151
+ the store root.
152
+
153
+ Prefix semantics (portable contract):
154
+ - prefix is a PATH prefix matched at slash boundaries, NOT a raw
155
+ substring. A slash-terminated prefix like "state/" selects every key
156
+ beneath the "state" directory and is the portable form that behaves
157
+ IDENTICALLY across all backends (local and cloud).
158
+ - An empty prefix lists everything.
159
+
160
+ A non-slash-terminated prefix (e.g. "state") is normalized to its
161
+ directory form before matching, so it selects keys under "state/" and
162
+ does NOT match siblings that merely share the leading characters (e.g.
163
+ "stateful/x.json"). Callers wanting portable results should pass a
164
+ slash-terminated prefix.
165
+ """
166
+
167
+ @abstractmethod
168
+ def delete(self, key: str) -> bool:
169
+ """
170
+ Delete key. Returns True if a value was removed, False if the key did
171
+ not exist (delete is idempotent and never raises on a missing key).
172
+ """
@@ -0,0 +1,305 @@
1
+ """
2
+ Optional cloud LokiStore backends: S3, GCS, Azure Blob.
3
+
4
+ Each backend's SDK is imported ONLY inside its __init__, so:
5
+ - The SDK is never imported at module import time.
6
+ - Local-only users never touch these classes and never pay an import cost.
7
+ - Selecting a cloud backend without its SDK installed raises a clear,
8
+ actionable BackendNotAvailableError naming the missing package.
9
+
10
+ Backends are intentionally thin. They do not add retry/credential frameworks;
11
+ they rely on each SDK's default credential chain (env vars, instance metadata,
12
+ etc.). Keys are normalized identically to LocalStore (traversal rejected) and
13
+ an optional store-level prefix is prepended to every object key.
14
+ """
15
+
16
+ from __future__ import annotations
17
+
18
+ import os
19
+ from pathlib import Path
20
+ from typing import List, Optional, Union
21
+
22
+ from .base import (
23
+ BackendNotAvailableError,
24
+ LokiStore,
25
+ normalize_key,
26
+ read_source_bytes,
27
+ )
28
+
29
+
30
+ def _join_prefix(prefix: str, key: str) -> str:
31
+ """Join a normalized store prefix and a normalized key into an object key."""
32
+ key = normalize_key(key)
33
+ if not prefix:
34
+ return key
35
+ return f"{prefix.rstrip('/')}/{key}"
36
+
37
+
38
+ def _normalize_prefix(prefix: Optional[str]) -> str:
39
+ """Normalize an optional store prefix to a clean POSIX-style path (or '')."""
40
+ if not prefix:
41
+ return ""
42
+ # Reuse the key guard so a prefix cannot smuggle traversal either.
43
+ return normalize_key(prefix)
44
+
45
+
46
+ def _dir_prefix(obj_prefix: str) -> str:
47
+ """
48
+ Coerce a non-empty object-key prefix to its directory form (slash
49
+ terminated) so cloud list() matches at slash boundaries like LocalStore's
50
+ directory walk, per the LokiStore.list contract. An empty prefix (list
51
+ everything) is returned unchanged.
52
+ """
53
+ if obj_prefix and not obj_prefix.endswith("/"):
54
+ return obj_prefix + "/"
55
+ return obj_prefix
56
+
57
+
58
+ class S3Store(LokiStore):
59
+ """Amazon S3 (or S3-compatible) backend (requires boto3)."""
60
+
61
+ def __init__(
62
+ self,
63
+ bucket: str,
64
+ prefix: Optional[str] = None,
65
+ region: Optional[str] = None,
66
+ ):
67
+ if not bucket:
68
+ raise ValueError("S3Store requires a bucket name")
69
+ try:
70
+ import boto3 # noqa: F401 (imported here so missing dep is lazy)
71
+ except ImportError as exc:
72
+ raise BackendNotAvailableError(
73
+ "The s3 storage backend requires the 'boto3' package. "
74
+ "Install it with: pip install boto3"
75
+ ) from exc
76
+
77
+ self._bucket = bucket
78
+ self._prefix = _normalize_prefix(prefix)
79
+ # boto3 picks up credentials from its default chain (env, shared
80
+ # config, instance/role metadata). region_name is optional.
81
+ kwargs = {"region_name": region} if region else {}
82
+ self._client = boto3.client("s3", **kwargs)
83
+
84
+ def _object_key(self, key: str) -> str:
85
+ return _join_prefix(self._prefix, key)
86
+
87
+ def put(self, key: str, data: Union[bytes, bytearray, str, os.PathLike]) -> None:
88
+ payload = read_source_bytes(data)
89
+ self._client.put_object(
90
+ Bucket=self._bucket, Key=self._object_key(key), Body=payload
91
+ )
92
+
93
+ def get(self, key: str) -> bytes:
94
+ try:
95
+ resp = self._client.get_object(
96
+ Bucket=self._bucket, Key=self._object_key(key)
97
+ )
98
+ except self._client.exceptions.NoSuchKey as exc:
99
+ raise FileNotFoundError(f"no such key: {key!r}") from exc
100
+ return resp["Body"].read()
101
+
102
+ def get_to(self, key: str, dest_path: Union[str, os.PathLike]) -> None:
103
+ dest = Path(dest_path).expanduser()
104
+ dest.parent.mkdir(parents=True, exist_ok=True)
105
+ self._client.download_file(self._bucket, self._object_key(key), str(dest))
106
+
107
+ def exists(self, key: str) -> bool:
108
+ try:
109
+ self._client.head_object(Bucket=self._bucket, Key=self._object_key(key))
110
+ return True
111
+ except Exception:
112
+ # head_object raises ClientError (404) when absent; treat any
113
+ # lookup failure as "not present" for the boolean contract.
114
+ return False
115
+
116
+ def list(self, prefix: str = "") -> List[str]:
117
+ # A caller prefix is matched at slash boundaries (directory form); the
118
+ # store-level prefix, when listing everything, is left as-is.
119
+ obj_prefix = _dir_prefix(self._object_key(prefix)) if prefix else self._prefix
120
+ paginator = self._client.get_paginator("list_objects_v2")
121
+ strip = (self._prefix + "/") if self._prefix else ""
122
+ keys: List[str] = []
123
+ for page in paginator.paginate(Bucket=self._bucket, Prefix=obj_prefix):
124
+ for item in page.get("Contents", []):
125
+ full = item["Key"]
126
+ rel = full[len(strip):] if strip and full.startswith(strip) else full
127
+ keys.append(rel)
128
+ keys.sort()
129
+ return keys
130
+
131
+ def delete(self, key: str) -> bool:
132
+ # S3 delete is idempotent and does not signal whether an object
133
+ # existed. Probe first so the boolean contract is honest.
134
+ existed = self.exists(key)
135
+ self._client.delete_object(Bucket=self._bucket, Key=self._object_key(key))
136
+ return existed
137
+
138
+
139
+ class GCSStore(LokiStore):
140
+ """Google Cloud Storage backend (requires google-cloud-storage)."""
141
+
142
+ def __init__(
143
+ self,
144
+ bucket: str,
145
+ prefix: Optional[str] = None,
146
+ region: Optional[str] = None, # accepted for signature parity, unused
147
+ ):
148
+ if not bucket:
149
+ raise ValueError("GCSStore requires a bucket name")
150
+ try:
151
+ from google.cloud import storage # type: ignore
152
+ except ImportError as exc:
153
+ raise BackendNotAvailableError(
154
+ "The gcs storage backend requires the 'google-cloud-storage' "
155
+ "package. Install it with: pip install google-cloud-storage"
156
+ ) from exc
157
+
158
+ self._prefix = _normalize_prefix(prefix)
159
+ # The client uses Application Default Credentials (env, gcloud, or
160
+ # workload identity).
161
+ self._client = storage.Client()
162
+ self._bucket = self._client.bucket(bucket)
163
+
164
+ def _object_key(self, key: str) -> str:
165
+ return _join_prefix(self._prefix, key)
166
+
167
+ def put(self, key: str, data: Union[bytes, bytearray, str, os.PathLike]) -> None:
168
+ payload = read_source_bytes(data)
169
+ blob = self._bucket.blob(self._object_key(key))
170
+ blob.upload_from_string(payload)
171
+
172
+ def get(self, key: str) -> bytes:
173
+ blob = self._bucket.blob(self._object_key(key))
174
+ if not blob.exists():
175
+ raise FileNotFoundError(f"no such key: {key!r}")
176
+ return blob.download_as_bytes()
177
+
178
+ def get_to(self, key: str, dest_path: Union[str, os.PathLike]) -> None:
179
+ dest = Path(dest_path).expanduser()
180
+ dest.parent.mkdir(parents=True, exist_ok=True)
181
+ blob = self._bucket.blob(self._object_key(key))
182
+ if not blob.exists():
183
+ raise FileNotFoundError(f"no such key: {key!r}")
184
+ blob.download_to_filename(str(dest))
185
+
186
+ def exists(self, key: str) -> bool:
187
+ return self._bucket.blob(self._object_key(key)).exists()
188
+
189
+ def list(self, prefix: str = "") -> List[str]:
190
+ obj_prefix = _dir_prefix(self._object_key(prefix)) if prefix else self._prefix
191
+ strip = (self._prefix + "/") if self._prefix else ""
192
+ keys: List[str] = []
193
+ for blob in self._client.list_blobs(self._bucket, prefix=obj_prefix):
194
+ full = blob.name
195
+ rel = full[len(strip):] if strip and full.startswith(strip) else full
196
+ keys.append(rel)
197
+ keys.sort()
198
+ return keys
199
+
200
+ def delete(self, key: str) -> bool:
201
+ blob = self._bucket.blob(self._object_key(key))
202
+ if not blob.exists():
203
+ return False
204
+ blob.delete()
205
+ return True
206
+
207
+
208
+ class AzureBlobStore(LokiStore):
209
+ """Azure Blob Storage backend (requires azure-storage-blob)."""
210
+
211
+ def __init__(
212
+ self,
213
+ bucket: str,
214
+ prefix: Optional[str] = None,
215
+ region: Optional[str] = None, # accepted for signature parity, unused
216
+ ):
217
+ # In Azure terms `bucket` is the container name.
218
+ if not bucket:
219
+ raise ValueError("AzureBlobStore requires a container name (bucket)")
220
+ try:
221
+ from azure.storage.blob import ContainerClient # type: ignore
222
+ except ImportError as exc:
223
+ raise BackendNotAvailableError(
224
+ "The azure-blob storage backend requires the "
225
+ "'azure-storage-blob' package. "
226
+ "Install it with: pip install azure-storage-blob"
227
+ ) from exc
228
+
229
+ self._prefix = _normalize_prefix(prefix)
230
+
231
+ # Credentials come from the SDK's default chain. We accept either a
232
+ # full connection string (AZURE_STORAGE_CONNECTION_STRING) or an
233
+ # account URL (AZURE_STORAGE_ACCOUNT_URL) + DefaultAzureCredential.
234
+ conn = os.environ.get("AZURE_STORAGE_CONNECTION_STRING")
235
+ account_url = os.environ.get("AZURE_STORAGE_ACCOUNT_URL")
236
+ if conn:
237
+ self._container = ContainerClient.from_connection_string(
238
+ conn, container_name=bucket
239
+ )
240
+ elif account_url:
241
+ try:
242
+ from azure.identity import DefaultAzureCredential # type: ignore
243
+ except ImportError as exc:
244
+ raise BackendNotAvailableError(
245
+ "Azure account-URL auth requires the 'azure-identity' "
246
+ "package. Install it with: pip install azure-identity"
247
+ ) from exc
248
+ self._container = ContainerClient(
249
+ account_url=account_url,
250
+ container_name=bucket,
251
+ credential=DefaultAzureCredential(),
252
+ )
253
+ else:
254
+ raise BackendNotAvailableError(
255
+ "The azure-blob backend needs AZURE_STORAGE_CONNECTION_STRING "
256
+ "or AZURE_STORAGE_ACCOUNT_URL to be set."
257
+ )
258
+
259
+ def _object_key(self, key: str) -> str:
260
+ return _join_prefix(self._prefix, key)
261
+
262
+ def put(self, key: str, data: Union[bytes, bytearray, str, os.PathLike]) -> None:
263
+ payload = read_source_bytes(data)
264
+ self._container.upload_blob(
265
+ name=self._object_key(key), data=payload, overwrite=True
266
+ )
267
+
268
+ def get(self, key: str) -> bytes:
269
+ from azure.core.exceptions import ResourceNotFoundError # type: ignore
270
+
271
+ try:
272
+ downloader = self._container.download_blob(self._object_key(key))
273
+ except ResourceNotFoundError as exc:
274
+ raise FileNotFoundError(f"no such key: {key!r}") from exc
275
+ return downloader.readall()
276
+
277
+ def get_to(self, key: str, dest_path: Union[str, os.PathLike]) -> None:
278
+ payload = self.get(key)
279
+ dest = Path(dest_path).expanduser()
280
+ dest.parent.mkdir(parents=True, exist_ok=True)
281
+ with open(dest, "wb") as f:
282
+ f.write(payload)
283
+
284
+ def exists(self, key: str) -> bool:
285
+ return self._container.get_blob_client(self._object_key(key)).exists()
286
+
287
+ def list(self, prefix: str = "") -> List[str]:
288
+ obj_prefix = _dir_prefix(self._object_key(prefix)) if prefix else self._prefix
289
+ strip = (self._prefix + "/") if self._prefix else ""
290
+ keys: List[str] = []
291
+ for blob in self._container.list_blobs(name_starts_with=obj_prefix or None):
292
+ full = blob.name
293
+ rel = full[len(strip):] if strip and full.startswith(strip) else full
294
+ keys.append(rel)
295
+ keys.sort()
296
+ return keys
297
+
298
+ def delete(self, key: str) -> bool:
299
+ from azure.core.exceptions import ResourceNotFoundError # type: ignore
300
+
301
+ try:
302
+ self._container.delete_blob(self._object_key(key))
303
+ return True
304
+ except ResourceNotFoundError:
305
+ return False