loki-mode 7.78.0 → 7.80.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/SKILL.md +2 -2
- package/VERSION +1 -1
- package/autonomy/lib/checkpoint_sync.py +189 -0
- package/autonomy/lib/config-map.sh +955 -0
- package/autonomy/loki +273 -8
- package/autonomy/run.sh +230 -195
- package/autonomy/sandbox.sh +98 -0
- package/autonomy/trigger-server.py +419 -43
- package/dashboard/__init__.py +1 -1
- package/dashboard/registry.py +212 -0
- package/dashboard/server.py +299 -7
- package/dashboard/static/index.html +383 -150
- package/docs/CONFIG-FILE-PLAN.md +459 -0
- package/docs/ENTERPRISE-IDENTITY-ROADMAP.md +206 -0
- package/docs/INSTALLATION.md +2 -2
- package/loki-ts/dist/loki.js +2 -2
- package/lokistore/__init__.py +65 -0
- package/lokistore/base.py +172 -0
- package/lokistore/cloud.py +305 -0
- package/lokistore/factory.py +187 -0
- package/lokistore/local.py +219 -0
- package/mcp/__init__.py +1 -1
- package/memory/retrieval.py +147 -0
- package/memory/tree_index.py +499 -0
- package/memory/tree_search.py +305 -0
- package/package.json +2 -1
- package/plugins/loki-mode/.claude-plugin/plugin.json +1 -1
package/docs/INSTALLATION.md
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
The flagship product of [Autonomi](https://www.autonomi.dev/). Loki Mode is a spec-driven autonomous builder with a built-in trust layer that takes any spec to a deployed product and verifies completion with evidence (quality gates plus a completion council), not just a "done" claim. Complete installation instructions for all platforms and use cases.
|
|
4
4
|
|
|
5
|
-
**Version:** v7.
|
|
5
|
+
**Version:** v7.80.0
|
|
6
6
|
|
|
7
7
|
---
|
|
8
8
|
|
|
@@ -395,7 +395,7 @@ provider works inside the container. Provide auth with your Anthropic API key:
|
|
|
395
395
|
# Run Loki Mode in Docker (Claude provider, API-key auth)
|
|
396
396
|
docker run --rm -e ANTHROPIC_API_KEY="$ANTHROPIC_API_KEY" \
|
|
397
397
|
-v $(pwd):/workspace -w /workspace \
|
|
398
|
-
asklokesh/loki-mode:7.
|
|
398
|
+
asklokesh/loki-mode:7.80.0 start ./my-spec.md
|
|
399
399
|
```
|
|
400
400
|
|
|
401
401
|
##### docker compose + .env (no host install)
|
package/loki-ts/dist/loki.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// @bun
|
|
2
|
-
var QQ=Object.defineProperty;var ZQ=($)=>$;function zQ($,Q){this[$]=ZQ.bind(null,Q)}var b=($,Q)=>{for(var Z in Q)QQ($,Z,{get:Q[Z],enumerable:!0,configurable:!0,set:zQ.bind(Q,Z)})};var L=($,Q)=>()=>($&&(Q=$($=0)),Q);var q$=import.meta.require;var h1={};b(h1,{lokiDir:()=>P,homeLokiDir:()=>i$,findRepoRootForVersion:()=>t$,REPO_ROOT:()=>g});import{resolve as a,dirname as r$}from"path";import{fileURLToPath as XQ}from"url";import{existsSync as R$}from"fs";import{homedir as KQ}from"os";function qQ(){let $=b1;for(let Q=0;Q<6;Q++){if(R$(a($,"VERSION"))&&R$(a($,"autonomy/run.sh")))return $;let Z=r$($);if(Z===$)break;$=Z}return a(b1,"..","..","..")}function t$($){let Q=$;for(let Z=0;Z<6;Z++){if(R$(a(Q,"VERSION"))&&R$(a(Q,"autonomy/run.sh")))return Q;let z=r$(Q);if(z===Q)break;Q=z}return a($,"..","..","..")}function P(){return process.env.LOKI_DIR??a(process.cwd(),".loki")}function i$(){return a(KQ(),".loki")}var b1,g;var C=L(()=>{b1=r$(XQ(import.meta.url));g=qQ()});import{readFileSync as VQ}from"fs";import{resolve as JQ,dirname as UQ}from"path";import{fileURLToPath as WQ}from"url";function E$(){if(Q$!==null)return Q$;let $="7.
|
|
2
|
+
var QQ=Object.defineProperty;var ZQ=($)=>$;function zQ($,Q){this[$]=ZQ.bind(null,Q)}var b=($,Q)=>{for(var Z in Q)QQ($,Z,{get:Q[Z],enumerable:!0,configurable:!0,set:zQ.bind(Q,Z)})};var L=($,Q)=>()=>($&&(Q=$($=0)),Q);var q$=import.meta.require;var h1={};b(h1,{lokiDir:()=>P,homeLokiDir:()=>i$,findRepoRootForVersion:()=>t$,REPO_ROOT:()=>g});import{resolve as a,dirname as r$}from"path";import{fileURLToPath as XQ}from"url";import{existsSync as R$}from"fs";import{homedir as KQ}from"os";function qQ(){let $=b1;for(let Q=0;Q<6;Q++){if(R$(a($,"VERSION"))&&R$(a($,"autonomy/run.sh")))return $;let Z=r$($);if(Z===$)break;$=Z}return a(b1,"..","..","..")}function t$($){let Q=$;for(let Z=0;Z<6;Z++){if(R$(a(Q,"VERSION"))&&R$(a(Q,"autonomy/run.sh")))return Q;let z=r$(Q);if(z===Q)break;Q=z}return a($,"..","..","..")}function P(){return process.env.LOKI_DIR??a(process.cwd(),".loki")}function i$(){return a(KQ(),".loki")}var b1,g;var C=L(()=>{b1=r$(XQ(import.meta.url));g=qQ()});import{readFileSync as VQ}from"fs";import{resolve as JQ,dirname as UQ}from"path";import{fileURLToPath as WQ}from"url";function E$(){if(Q$!==null)return Q$;let $="7.80.0";if(typeof $==="string"&&$.length>0)return Q$=$,Q$;try{let Q=UQ(WQ(import.meta.url)),Z=t$(Q);Q$=VQ(JQ(Z,"VERSION"),"utf-8").trim()}catch{Q$="unknown"}return Q$}var Q$=null;var e$=L(()=>{C()});var g1={};b(g1,{runOrThrow:()=>HQ,run:()=>F,readStreamCapped:()=>m1,commandVersion:()=>BQ,commandExists:()=>f,ShellError:()=>$1,MAX_STDOUT_BYTES:()=>v1});async function m1($,Q=v1){let Z=$.getReader(),z=new TextDecoder,X="",q=0;try{while(q<Q){let{done:K,value:U}=await Z.read();if(K)break;if(!U)continue;if(q+=U.byteLength,q>Q){let J=U.byteLength-(q-Q);X+=z.decode(U.subarray(0,J),{stream:!0});break}X+=z.decode(U,{stream:!0})}X+=z.decode()}finally{try{await Z.cancel()}catch{}Z.releaseLock()}return X}async function F($,Q={}){let Z=Bun.spawn({cmd:[...$],stdout:"pipe",stderr:"pipe",env:Q.env?{...process.env,...Q.env}:process.env,cwd:Q.cwd}),z,X;if(Q.timeoutMs&&Q.timeoutMs>0)z=setTimeout(()=>{try{Z.kill("SIGTERM")}catch{}X=setTimeout(()=>{try{Z.kill("SIGKILL")}catch{}},2000)},Q.timeoutMs);try{let[q,K,U]=await Promise.all([m1(Z.stdout),new Response(Z.stderr).text(),Z.exited]);return{stdout:q,stderr:K,exitCode:U}}finally{if(z)clearTimeout(z);if(X)clearTimeout(X)}}async function HQ($,Q={}){let Z=await F($,Q);if(Z.exitCode!==0)throw new $1(`command failed (${Z.exitCode}): ${$.join(" ")}`,Z.exitCode,Z.stdout,Z.stderr);return Z}async function f($){let Q=GQ($),Z=await F(["sh","-c",`command -v ${Q}`],{timeoutMs:5000});if(Z.exitCode===0)return Z.stdout.trim()||null;return null}function GQ($){if(!/^[A-Za-z0-9._/-]+$/.test($))throw Error(`refused to shell-escape suspect token: ${$}`);return $}async function BQ($,Q="--version"){if(!await f($))return null;let z=await F([$,Q],{timeoutMs:5000});if(z.exitCode!==0)return null;return((z.stdout||z.stderr).split(/\r?\n/)[0]?.trim()??"")||null}var v1=16777216,$1;var d=L(()=>{$1=class $1 extends Error{message;exitCode;stdout;stderr;constructor($,Q,Z,z){super($);this.message=$;this.exitCode=Q;this.stdout=Z;this.stderr=z;this.name="ShellError"}}});function s($){return YQ?"":$}var YQ,O,S,_,_Z,I,k,h,V;var c=L(()=>{YQ=(process.env.NO_COLOR??"").length>0;O=s("\x1B[0;31m"),S=s("\x1B[0;32m"),_=s("\x1B[1;33m"),_Z=s("\x1B[0;34m"),I=s("\x1B[0;36m"),k=s("\x1B[1m"),h=s("\x1B[2m"),V=s("\x1B[0m")});import{existsSync as jQ}from"fs";async function Z$(){if(Y$!==void 0)return Y$;let $="/opt/homebrew/bin/python3.12";if(jQ($))return Y$=$,$;let Q=await f("python3.12");if(Q)return Y$=Q,Q;let Z=await f("python3");return Y$=Z,Z}async function z$($,Q={}){let Z=await Z$();if(!Z)return{stdout:"",stderr:"python3 not found",exitCode:127};return F([Z,"-c",$],Q)}var Y$;var V$=L(()=>{d()});var X0={};b(X0,{runStatus:()=>oQ});import{existsSync as y,readFileSync as U$,readdirSync as r1,statSync as t1}from"fs";import{resolve as D,basename as vQ}from"path";import{homedir as mQ}from"os";function i1($){let Q=Math.trunc($);if(Q>=1e6)return`${(Math.trunc(Q/1e6*10)/10).toFixed(1)}M`;if(Q>=1000)return`${(Math.trunc(Q/1000*10)/10).toFixed(1)}K`;return String(Q)}function e1($,Q,Z){if(Q===0)return null;let z=Math.trunc($*100/Q),X=Math.trunc($*N$/Q);if(X>N$)X=N$;let q=N$-X,K=S;if(z>=80)K=O;else if(z>=50)K=_;let U="=".repeat(Math.max(0,X))+" ".repeat(Math.max(0,q)),J=i1($),W=i1(Q);return` ${k}${Z}${V} ${K}[${U}]${V} ${z}% (${J} / ${W})`}async function fQ(){if(await f("jq"))return!0;return process.stdout.write(`${O}Error: jq is required but not installed.${V}
|
|
3
3
|
`),process.stdout.write(`Install with:
|
|
4
4
|
`),process.stdout.write(` brew install jq (macOS)
|
|
5
5
|
`),process.stdout.write(` apt install jq (Debian/Ubuntu)
|
|
@@ -793,4 +793,4 @@ Set LOKI_LEGACY_BASH=1 to force the bash CLI for every command.
|
|
|
793
793
|
`),2}default:return process.stderr.write(`Unknown command: ${Q}
|
|
794
794
|
`),process.stderr.write($Q),2}}s1();process.on("SIGINT",()=>process.exit(130));process.on("SIGTERM",()=>process.exit(143));var qZ=await KZ(Bun.argv.slice(2));process.exit(qZ);
|
|
795
795
|
|
|
796
|
-
//# debugId=
|
|
796
|
+
//# debugId=5DBFF3AC2E319A5664756E2164756E21
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
"""
|
|
2
|
+
LokiStore: pluggable, local-first storage abstraction for Loki Mode.
|
|
3
|
+
|
|
4
|
+
This package is the consolidation spine for durable artifacts (state,
|
|
5
|
+
checkpoints, memory blobs, healing artifacts, etc.). Features should bind to
|
|
6
|
+
this abstraction instead of hardcoding `.loki/` paths or embedding their own
|
|
7
|
+
cloud SDK clients.
|
|
8
|
+
|
|
9
|
+
Design goals
|
|
10
|
+
------------
|
|
11
|
+
- Local-first by default. With nothing configured, LocalStore writes to the
|
|
12
|
+
project `.loki/` directory with byte-identical behavior to today's direct
|
|
13
|
+
file writes (atomic temp+rename, fcntl advisory locking, path-traversal
|
|
14
|
+
guard). Zero new dependencies, zero behavior change for local users.
|
|
15
|
+
- Optional cloud backends (S3, GCS, Azure Blob) behind lazy imports. The
|
|
16
|
+
cloud SDK is imported ONLY inside the backend's __init__, so a missing
|
|
17
|
+
dependency raises a clear, actionable error ONLY when that backend is
|
|
18
|
+
explicitly selected -- never at import time, never for local users.
|
|
19
|
+
- One interface (put/get/get_to/exists/list/delete) so callers are backend
|
|
20
|
+
agnostic.
|
|
21
|
+
|
|
22
|
+
Quick start
|
|
23
|
+
-----------
|
|
24
|
+
from lokistore import get_store
|
|
25
|
+
|
|
26
|
+
store = get_store() # local by default, honors LOKI_DIR/TARGET_DIR
|
|
27
|
+
store.put("state/checkpoints/cp-1/metadata.json", b"{...}")
|
|
28
|
+
data = store.get("state/checkpoints/cp-1/metadata.json")
|
|
29
|
+
for key in store.list("state/checkpoints/"):
|
|
30
|
+
...
|
|
31
|
+
|
|
32
|
+
Configuration (env or config dict)
|
|
33
|
+
----------------------------------
|
|
34
|
+
- LOKI_STORAGE_BACKEND : local | s3 | gcs | azure-blob (default: local)
|
|
35
|
+
- LOKI_STORAGE_BUCKET : bucket / container name (cloud backends)
|
|
36
|
+
- LOKI_STORAGE_PREFIX : key prefix within the bucket (optional)
|
|
37
|
+
- LOKI_STORAGE_REGION : region (s3, optional)
|
|
38
|
+
|
|
39
|
+
The metadata backend (sqlite default, postgres later) is selected separately;
|
|
40
|
+
see get_metadata_backend() / METADATA notes below. For this release the blob
|
|
41
|
+
backends are the core deliverable and the metadata selector is a thin,
|
|
42
|
+
documented stub that defaults to the existing sqlite path.
|
|
43
|
+
"""
|
|
44
|
+
|
|
45
|
+
from .base import LokiStore, StoreError, BackendNotAvailableError
|
|
46
|
+
from .local import LocalStore
|
|
47
|
+
from .factory import (
|
|
48
|
+
get_store,
|
|
49
|
+
build_store,
|
|
50
|
+
get_metadata_backend,
|
|
51
|
+
resolve_local_base,
|
|
52
|
+
)
|
|
53
|
+
|
|
54
|
+
__all__ = [
|
|
55
|
+
"LokiStore",
|
|
56
|
+
"StoreError",
|
|
57
|
+
"BackendNotAvailableError",
|
|
58
|
+
"LocalStore",
|
|
59
|
+
"get_store",
|
|
60
|
+
"build_store",
|
|
61
|
+
"get_metadata_backend",
|
|
62
|
+
"resolve_local_base",
|
|
63
|
+
]
|
|
64
|
+
|
|
65
|
+
__version__ = "1.0.0"
|
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
"""
|
|
2
|
+
LokiStore interface and shared helpers.
|
|
3
|
+
|
|
4
|
+
Defines the abstract blob-store contract that every backend implements, plus
|
|
5
|
+
errors and a key-normalization helper shared across backends.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import os
|
|
11
|
+
from abc import ABC, abstractmethod
|
|
12
|
+
from typing import List, Union
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class StoreError(Exception):
|
|
16
|
+
"""Base class for LokiStore errors."""
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class BackendNotAvailableError(StoreError):
|
|
20
|
+
"""
|
|
21
|
+
Raised when a cloud backend is explicitly selected but its SDK (or
|
|
22
|
+
required configuration) is not available. The message names the missing
|
|
23
|
+
package and how to install it so the failure is actionable.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def normalize_key(key: str) -> str:
|
|
28
|
+
"""
|
|
29
|
+
Normalize a store key to a clean POSIX-style relative path.
|
|
30
|
+
|
|
31
|
+
Keys are relative paths like "state/checkpoints/cp-1/metadata.json".
|
|
32
|
+
This rejects absolute paths and any key that would traverse outside the
|
|
33
|
+
store root (a leading or embedded ".." segment). The same guard is used
|
|
34
|
+
by every backend so traversal is rejected uniformly, whether the bytes
|
|
35
|
+
land on a local filesystem or in an object-store key namespace.
|
|
36
|
+
|
|
37
|
+
Args:
|
|
38
|
+
key: A POSIX-style relative path.
|
|
39
|
+
|
|
40
|
+
Returns:
|
|
41
|
+
The normalized key (forward slashes, no leading slash, no "."/".."
|
|
42
|
+
segments collapsed away).
|
|
43
|
+
|
|
44
|
+
Raises:
|
|
45
|
+
ValueError: If the key is empty, absolute, or escapes the root.
|
|
46
|
+
"""
|
|
47
|
+
if not isinstance(key, str) or not key:
|
|
48
|
+
raise ValueError("store key must be a non-empty string")
|
|
49
|
+
|
|
50
|
+
# Treat backslashes as separators too, so a Windows-style key cannot
|
|
51
|
+
# smuggle a traversal past the split() check below.
|
|
52
|
+
candidate = key.replace("\\", "/")
|
|
53
|
+
|
|
54
|
+
if candidate.startswith("/"):
|
|
55
|
+
raise ValueError(f"absolute keys are not allowed: {key!r}")
|
|
56
|
+
|
|
57
|
+
parts: List[str] = []
|
|
58
|
+
for segment in candidate.split("/"):
|
|
59
|
+
if segment == "" or segment == ".":
|
|
60
|
+
# Collapse empty (e.g. "a//b") and current-dir segments.
|
|
61
|
+
continue
|
|
62
|
+
if segment == "..":
|
|
63
|
+
raise ValueError(f"path traversal is not allowed in key: {key!r}")
|
|
64
|
+
parts.append(segment)
|
|
65
|
+
|
|
66
|
+
if not parts:
|
|
67
|
+
raise ValueError(f"key resolves to an empty path: {key!r}")
|
|
68
|
+
|
|
69
|
+
return "/".join(parts)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def read_source_bytes(data: Union[bytes, bytearray, str, os.PathLike]) -> bytes:
|
|
73
|
+
"""
|
|
74
|
+
Coerce a put() source argument into bytes.
|
|
75
|
+
|
|
76
|
+
Accepts raw bytes (returned as-is) or a path-like pointing at a file whose
|
|
77
|
+
contents are read in binary. A plain str is treated as a FILESYSTEM PATH,
|
|
78
|
+
not as text, to match the put(key, bytes_or_path) contract; callers that
|
|
79
|
+
want to store a string must encode it themselves.
|
|
80
|
+
|
|
81
|
+
Args:
|
|
82
|
+
data: bytes/bytearray, or a path-like to an existing file.
|
|
83
|
+
|
|
84
|
+
Returns:
|
|
85
|
+
The bytes to store.
|
|
86
|
+
|
|
87
|
+
Raises:
|
|
88
|
+
FileNotFoundError: If a path is given and the file does not exist.
|
|
89
|
+
TypeError: If the argument is neither bytes nor a path-like.
|
|
90
|
+
"""
|
|
91
|
+
if isinstance(data, (bytes, bytearray)):
|
|
92
|
+
return bytes(data)
|
|
93
|
+
if isinstance(data, (str, os.PathLike)):
|
|
94
|
+
with open(data, "rb") as f:
|
|
95
|
+
return f.read()
|
|
96
|
+
raise TypeError(
|
|
97
|
+
"put() source must be bytes or a path-like to a file, "
|
|
98
|
+
f"got {type(data).__name__}"
|
|
99
|
+
)
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
class LokiStore(ABC):
|
|
103
|
+
"""
|
|
104
|
+
Abstract pluggable blob store.
|
|
105
|
+
|
|
106
|
+
All keys are POSIX-style relative paths (see normalize_key). Implementations
|
|
107
|
+
must apply normalize_key to every incoming key so traversal is rejected and
|
|
108
|
+
behavior is uniform across backends.
|
|
109
|
+
|
|
110
|
+
The contract is intentionally small (no streaming, no multipart, no
|
|
111
|
+
credential framework) so backends stay thin. Larger payload handling can be
|
|
112
|
+
layered on later without changing this interface.
|
|
113
|
+
"""
|
|
114
|
+
|
|
115
|
+
@abstractmethod
|
|
116
|
+
def put(self, key: str, data: Union[bytes, bytearray, str, os.PathLike]) -> None:
|
|
117
|
+
"""
|
|
118
|
+
Store bytes under key. The source is either raw bytes or a path-like
|
|
119
|
+
to a file whose contents are stored. Overwrites any existing value
|
|
120
|
+
atomically where the backend supports it.
|
|
121
|
+
"""
|
|
122
|
+
|
|
123
|
+
@abstractmethod
|
|
124
|
+
def get(self, key: str) -> bytes:
|
|
125
|
+
"""
|
|
126
|
+
Return the bytes stored under key.
|
|
127
|
+
|
|
128
|
+
Raises:
|
|
129
|
+
FileNotFoundError: If the key does not exist.
|
|
130
|
+
"""
|
|
131
|
+
|
|
132
|
+
@abstractmethod
|
|
133
|
+
def get_to(self, key: str, dest_path: Union[str, os.PathLike]) -> None:
|
|
134
|
+
"""
|
|
135
|
+
Write the bytes stored under key to dest_path (a local filesystem
|
|
136
|
+
path). Parent directories are created as needed.
|
|
137
|
+
|
|
138
|
+
Raises:
|
|
139
|
+
FileNotFoundError: If the key does not exist.
|
|
140
|
+
"""
|
|
141
|
+
|
|
142
|
+
@abstractmethod
|
|
143
|
+
def exists(self, key: str) -> bool:
|
|
144
|
+
"""Return True if key exists in the store."""
|
|
145
|
+
|
|
146
|
+
@abstractmethod
|
|
147
|
+
def list(self, prefix: str = "") -> List[str]:
|
|
148
|
+
"""
|
|
149
|
+
List keys under prefix (a POSIX-style relative PATH prefix). An empty
|
|
150
|
+
prefix lists every key. Returns normalized keys, sorted, relative to
|
|
151
|
+
the store root.
|
|
152
|
+
|
|
153
|
+
Prefix semantics (portable contract):
|
|
154
|
+
- prefix is a PATH prefix matched at slash boundaries, NOT a raw
|
|
155
|
+
substring. A slash-terminated prefix like "state/" selects every key
|
|
156
|
+
beneath the "state" directory and is the portable form that behaves
|
|
157
|
+
IDENTICALLY across all backends (local and cloud).
|
|
158
|
+
- An empty prefix lists everything.
|
|
159
|
+
|
|
160
|
+
A non-slash-terminated prefix (e.g. "state") is normalized to its
|
|
161
|
+
directory form before matching, so it selects keys under "state/" and
|
|
162
|
+
does NOT match siblings that merely share the leading characters (e.g.
|
|
163
|
+
"stateful/x.json"). Callers wanting portable results should pass a
|
|
164
|
+
slash-terminated prefix.
|
|
165
|
+
"""
|
|
166
|
+
|
|
167
|
+
@abstractmethod
|
|
168
|
+
def delete(self, key: str) -> bool:
|
|
169
|
+
"""
|
|
170
|
+
Delete key. Returns True if a value was removed, False if the key did
|
|
171
|
+
not exist (delete is idempotent and never raises on a missing key).
|
|
172
|
+
"""
|
|
@@ -0,0 +1,305 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Optional cloud LokiStore backends: S3, GCS, Azure Blob.
|
|
3
|
+
|
|
4
|
+
Each backend's SDK is imported ONLY inside its __init__, so:
|
|
5
|
+
- The SDK is never imported at module import time.
|
|
6
|
+
- Local-only users never touch these classes and never pay an import cost.
|
|
7
|
+
- Selecting a cloud backend without its SDK installed raises a clear,
|
|
8
|
+
actionable BackendNotAvailableError naming the missing package.
|
|
9
|
+
|
|
10
|
+
Backends are intentionally thin. They do not add retry/credential frameworks;
|
|
11
|
+
they rely on each SDK's default credential chain (env vars, instance metadata,
|
|
12
|
+
etc.). Keys are normalized identically to LocalStore (traversal rejected) and
|
|
13
|
+
an optional store-level prefix is prepended to every object key.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import os
|
|
19
|
+
from pathlib import Path
|
|
20
|
+
from typing import List, Optional, Union
|
|
21
|
+
|
|
22
|
+
from .base import (
|
|
23
|
+
BackendNotAvailableError,
|
|
24
|
+
LokiStore,
|
|
25
|
+
normalize_key,
|
|
26
|
+
read_source_bytes,
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _join_prefix(prefix: str, key: str) -> str:
|
|
31
|
+
"""Join a normalized store prefix and a normalized key into an object key."""
|
|
32
|
+
key = normalize_key(key)
|
|
33
|
+
if not prefix:
|
|
34
|
+
return key
|
|
35
|
+
return f"{prefix.rstrip('/')}/{key}"
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _normalize_prefix(prefix: Optional[str]) -> str:
|
|
39
|
+
"""Normalize an optional store prefix to a clean POSIX-style path (or '')."""
|
|
40
|
+
if not prefix:
|
|
41
|
+
return ""
|
|
42
|
+
# Reuse the key guard so a prefix cannot smuggle traversal either.
|
|
43
|
+
return normalize_key(prefix)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _dir_prefix(obj_prefix: str) -> str:
|
|
47
|
+
"""
|
|
48
|
+
Coerce a non-empty object-key prefix to its directory form (slash
|
|
49
|
+
terminated) so cloud list() matches at slash boundaries like LocalStore's
|
|
50
|
+
directory walk, per the LokiStore.list contract. An empty prefix (list
|
|
51
|
+
everything) is returned unchanged.
|
|
52
|
+
"""
|
|
53
|
+
if obj_prefix and not obj_prefix.endswith("/"):
|
|
54
|
+
return obj_prefix + "/"
|
|
55
|
+
return obj_prefix
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
class S3Store(LokiStore):
|
|
59
|
+
"""Amazon S3 (or S3-compatible) backend (requires boto3)."""
|
|
60
|
+
|
|
61
|
+
def __init__(
|
|
62
|
+
self,
|
|
63
|
+
bucket: str,
|
|
64
|
+
prefix: Optional[str] = None,
|
|
65
|
+
region: Optional[str] = None,
|
|
66
|
+
):
|
|
67
|
+
if not bucket:
|
|
68
|
+
raise ValueError("S3Store requires a bucket name")
|
|
69
|
+
try:
|
|
70
|
+
import boto3 # noqa: F401 (imported here so missing dep is lazy)
|
|
71
|
+
except ImportError as exc:
|
|
72
|
+
raise BackendNotAvailableError(
|
|
73
|
+
"The s3 storage backend requires the 'boto3' package. "
|
|
74
|
+
"Install it with: pip install boto3"
|
|
75
|
+
) from exc
|
|
76
|
+
|
|
77
|
+
self._bucket = bucket
|
|
78
|
+
self._prefix = _normalize_prefix(prefix)
|
|
79
|
+
# boto3 picks up credentials from its default chain (env, shared
|
|
80
|
+
# config, instance/role metadata). region_name is optional.
|
|
81
|
+
kwargs = {"region_name": region} if region else {}
|
|
82
|
+
self._client = boto3.client("s3", **kwargs)
|
|
83
|
+
|
|
84
|
+
def _object_key(self, key: str) -> str:
|
|
85
|
+
return _join_prefix(self._prefix, key)
|
|
86
|
+
|
|
87
|
+
def put(self, key: str, data: Union[bytes, bytearray, str, os.PathLike]) -> None:
|
|
88
|
+
payload = read_source_bytes(data)
|
|
89
|
+
self._client.put_object(
|
|
90
|
+
Bucket=self._bucket, Key=self._object_key(key), Body=payload
|
|
91
|
+
)
|
|
92
|
+
|
|
93
|
+
def get(self, key: str) -> bytes:
|
|
94
|
+
try:
|
|
95
|
+
resp = self._client.get_object(
|
|
96
|
+
Bucket=self._bucket, Key=self._object_key(key)
|
|
97
|
+
)
|
|
98
|
+
except self._client.exceptions.NoSuchKey as exc:
|
|
99
|
+
raise FileNotFoundError(f"no such key: {key!r}") from exc
|
|
100
|
+
return resp["Body"].read()
|
|
101
|
+
|
|
102
|
+
def get_to(self, key: str, dest_path: Union[str, os.PathLike]) -> None:
|
|
103
|
+
dest = Path(dest_path).expanduser()
|
|
104
|
+
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
105
|
+
self._client.download_file(self._bucket, self._object_key(key), str(dest))
|
|
106
|
+
|
|
107
|
+
def exists(self, key: str) -> bool:
|
|
108
|
+
try:
|
|
109
|
+
self._client.head_object(Bucket=self._bucket, Key=self._object_key(key))
|
|
110
|
+
return True
|
|
111
|
+
except Exception:
|
|
112
|
+
# head_object raises ClientError (404) when absent; treat any
|
|
113
|
+
# lookup failure as "not present" for the boolean contract.
|
|
114
|
+
return False
|
|
115
|
+
|
|
116
|
+
def list(self, prefix: str = "") -> List[str]:
|
|
117
|
+
# A caller prefix is matched at slash boundaries (directory form); the
|
|
118
|
+
# store-level prefix, when listing everything, is left as-is.
|
|
119
|
+
obj_prefix = _dir_prefix(self._object_key(prefix)) if prefix else self._prefix
|
|
120
|
+
paginator = self._client.get_paginator("list_objects_v2")
|
|
121
|
+
strip = (self._prefix + "/") if self._prefix else ""
|
|
122
|
+
keys: List[str] = []
|
|
123
|
+
for page in paginator.paginate(Bucket=self._bucket, Prefix=obj_prefix):
|
|
124
|
+
for item in page.get("Contents", []):
|
|
125
|
+
full = item["Key"]
|
|
126
|
+
rel = full[len(strip):] if strip and full.startswith(strip) else full
|
|
127
|
+
keys.append(rel)
|
|
128
|
+
keys.sort()
|
|
129
|
+
return keys
|
|
130
|
+
|
|
131
|
+
def delete(self, key: str) -> bool:
|
|
132
|
+
# S3 delete is idempotent and does not signal whether an object
|
|
133
|
+
# existed. Probe first so the boolean contract is honest.
|
|
134
|
+
existed = self.exists(key)
|
|
135
|
+
self._client.delete_object(Bucket=self._bucket, Key=self._object_key(key))
|
|
136
|
+
return existed
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
class GCSStore(LokiStore):
|
|
140
|
+
"""Google Cloud Storage backend (requires google-cloud-storage)."""
|
|
141
|
+
|
|
142
|
+
def __init__(
|
|
143
|
+
self,
|
|
144
|
+
bucket: str,
|
|
145
|
+
prefix: Optional[str] = None,
|
|
146
|
+
region: Optional[str] = None, # accepted for signature parity, unused
|
|
147
|
+
):
|
|
148
|
+
if not bucket:
|
|
149
|
+
raise ValueError("GCSStore requires a bucket name")
|
|
150
|
+
try:
|
|
151
|
+
from google.cloud import storage # type: ignore
|
|
152
|
+
except ImportError as exc:
|
|
153
|
+
raise BackendNotAvailableError(
|
|
154
|
+
"The gcs storage backend requires the 'google-cloud-storage' "
|
|
155
|
+
"package. Install it with: pip install google-cloud-storage"
|
|
156
|
+
) from exc
|
|
157
|
+
|
|
158
|
+
self._prefix = _normalize_prefix(prefix)
|
|
159
|
+
# The client uses Application Default Credentials (env, gcloud, or
|
|
160
|
+
# workload identity).
|
|
161
|
+
self._client = storage.Client()
|
|
162
|
+
self._bucket = self._client.bucket(bucket)
|
|
163
|
+
|
|
164
|
+
def _object_key(self, key: str) -> str:
|
|
165
|
+
return _join_prefix(self._prefix, key)
|
|
166
|
+
|
|
167
|
+
def put(self, key: str, data: Union[bytes, bytearray, str, os.PathLike]) -> None:
|
|
168
|
+
payload = read_source_bytes(data)
|
|
169
|
+
blob = self._bucket.blob(self._object_key(key))
|
|
170
|
+
blob.upload_from_string(payload)
|
|
171
|
+
|
|
172
|
+
def get(self, key: str) -> bytes:
|
|
173
|
+
blob = self._bucket.blob(self._object_key(key))
|
|
174
|
+
if not blob.exists():
|
|
175
|
+
raise FileNotFoundError(f"no such key: {key!r}")
|
|
176
|
+
return blob.download_as_bytes()
|
|
177
|
+
|
|
178
|
+
def get_to(self, key: str, dest_path: Union[str, os.PathLike]) -> None:
|
|
179
|
+
dest = Path(dest_path).expanduser()
|
|
180
|
+
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
181
|
+
blob = self._bucket.blob(self._object_key(key))
|
|
182
|
+
if not blob.exists():
|
|
183
|
+
raise FileNotFoundError(f"no such key: {key!r}")
|
|
184
|
+
blob.download_to_filename(str(dest))
|
|
185
|
+
|
|
186
|
+
def exists(self, key: str) -> bool:
|
|
187
|
+
return self._bucket.blob(self._object_key(key)).exists()
|
|
188
|
+
|
|
189
|
+
def list(self, prefix: str = "") -> List[str]:
|
|
190
|
+
obj_prefix = _dir_prefix(self._object_key(prefix)) if prefix else self._prefix
|
|
191
|
+
strip = (self._prefix + "/") if self._prefix else ""
|
|
192
|
+
keys: List[str] = []
|
|
193
|
+
for blob in self._client.list_blobs(self._bucket, prefix=obj_prefix):
|
|
194
|
+
full = blob.name
|
|
195
|
+
rel = full[len(strip):] if strip and full.startswith(strip) else full
|
|
196
|
+
keys.append(rel)
|
|
197
|
+
keys.sort()
|
|
198
|
+
return keys
|
|
199
|
+
|
|
200
|
+
def delete(self, key: str) -> bool:
|
|
201
|
+
blob = self._bucket.blob(self._object_key(key))
|
|
202
|
+
if not blob.exists():
|
|
203
|
+
return False
|
|
204
|
+
blob.delete()
|
|
205
|
+
return True
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
class AzureBlobStore(LokiStore):
|
|
209
|
+
"""Azure Blob Storage backend (requires azure-storage-blob)."""
|
|
210
|
+
|
|
211
|
+
def __init__(
|
|
212
|
+
self,
|
|
213
|
+
bucket: str,
|
|
214
|
+
prefix: Optional[str] = None,
|
|
215
|
+
region: Optional[str] = None, # accepted for signature parity, unused
|
|
216
|
+
):
|
|
217
|
+
# In Azure terms `bucket` is the container name.
|
|
218
|
+
if not bucket:
|
|
219
|
+
raise ValueError("AzureBlobStore requires a container name (bucket)")
|
|
220
|
+
try:
|
|
221
|
+
from azure.storage.blob import ContainerClient # type: ignore
|
|
222
|
+
except ImportError as exc:
|
|
223
|
+
raise BackendNotAvailableError(
|
|
224
|
+
"The azure-blob storage backend requires the "
|
|
225
|
+
"'azure-storage-blob' package. "
|
|
226
|
+
"Install it with: pip install azure-storage-blob"
|
|
227
|
+
) from exc
|
|
228
|
+
|
|
229
|
+
self._prefix = _normalize_prefix(prefix)
|
|
230
|
+
|
|
231
|
+
# Credentials come from the SDK's default chain. We accept either a
|
|
232
|
+
# full connection string (AZURE_STORAGE_CONNECTION_STRING) or an
|
|
233
|
+
# account URL (AZURE_STORAGE_ACCOUNT_URL) + DefaultAzureCredential.
|
|
234
|
+
conn = os.environ.get("AZURE_STORAGE_CONNECTION_STRING")
|
|
235
|
+
account_url = os.environ.get("AZURE_STORAGE_ACCOUNT_URL")
|
|
236
|
+
if conn:
|
|
237
|
+
self._container = ContainerClient.from_connection_string(
|
|
238
|
+
conn, container_name=bucket
|
|
239
|
+
)
|
|
240
|
+
elif account_url:
|
|
241
|
+
try:
|
|
242
|
+
from azure.identity import DefaultAzureCredential # type: ignore
|
|
243
|
+
except ImportError as exc:
|
|
244
|
+
raise BackendNotAvailableError(
|
|
245
|
+
"Azure account-URL auth requires the 'azure-identity' "
|
|
246
|
+
"package. Install it with: pip install azure-identity"
|
|
247
|
+
) from exc
|
|
248
|
+
self._container = ContainerClient(
|
|
249
|
+
account_url=account_url,
|
|
250
|
+
container_name=bucket,
|
|
251
|
+
credential=DefaultAzureCredential(),
|
|
252
|
+
)
|
|
253
|
+
else:
|
|
254
|
+
raise BackendNotAvailableError(
|
|
255
|
+
"The azure-blob backend needs AZURE_STORAGE_CONNECTION_STRING "
|
|
256
|
+
"or AZURE_STORAGE_ACCOUNT_URL to be set."
|
|
257
|
+
)
|
|
258
|
+
|
|
259
|
+
def _object_key(self, key: str) -> str:
|
|
260
|
+
return _join_prefix(self._prefix, key)
|
|
261
|
+
|
|
262
|
+
def put(self, key: str, data: Union[bytes, bytearray, str, os.PathLike]) -> None:
|
|
263
|
+
payload = read_source_bytes(data)
|
|
264
|
+
self._container.upload_blob(
|
|
265
|
+
name=self._object_key(key), data=payload, overwrite=True
|
|
266
|
+
)
|
|
267
|
+
|
|
268
|
+
def get(self, key: str) -> bytes:
|
|
269
|
+
from azure.core.exceptions import ResourceNotFoundError # type: ignore
|
|
270
|
+
|
|
271
|
+
try:
|
|
272
|
+
downloader = self._container.download_blob(self._object_key(key))
|
|
273
|
+
except ResourceNotFoundError as exc:
|
|
274
|
+
raise FileNotFoundError(f"no such key: {key!r}") from exc
|
|
275
|
+
return downloader.readall()
|
|
276
|
+
|
|
277
|
+
def get_to(self, key: str, dest_path: Union[str, os.PathLike]) -> None:
|
|
278
|
+
payload = self.get(key)
|
|
279
|
+
dest = Path(dest_path).expanduser()
|
|
280
|
+
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
281
|
+
with open(dest, "wb") as f:
|
|
282
|
+
f.write(payload)
|
|
283
|
+
|
|
284
|
+
def exists(self, key: str) -> bool:
|
|
285
|
+
return self._container.get_blob_client(self._object_key(key)).exists()
|
|
286
|
+
|
|
287
|
+
def list(self, prefix: str = "") -> List[str]:
|
|
288
|
+
obj_prefix = _dir_prefix(self._object_key(prefix)) if prefix else self._prefix
|
|
289
|
+
strip = (self._prefix + "/") if self._prefix else ""
|
|
290
|
+
keys: List[str] = []
|
|
291
|
+
for blob in self._container.list_blobs(name_starts_with=obj_prefix or None):
|
|
292
|
+
full = blob.name
|
|
293
|
+
rel = full[len(strip):] if strip and full.startswith(strip) else full
|
|
294
|
+
keys.append(rel)
|
|
295
|
+
keys.sort()
|
|
296
|
+
return keys
|
|
297
|
+
|
|
298
|
+
def delete(self, key: str) -> bool:
|
|
299
|
+
from azure.core.exceptions import ResourceNotFoundError # type: ignore
|
|
300
|
+
|
|
301
|
+
try:
|
|
302
|
+
self._container.delete_blob(self._object_key(key))
|
|
303
|
+
return True
|
|
304
|
+
except ResourceNotFoundError:
|
|
305
|
+
return False
|