@knolo/cli 3.2.2 → 5.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +68 -7
- package/bin/knolo.mjs +514 -7
- package/package.json +21 -3
- package/templates/icp-knowledge-canister/README.md +76 -0
- package/templates/icp-knowledge-canister/canisters/knolo-core-rust/Cargo.toml +13 -0
- package/templates/icp-knowledge-canister/canisters/knolo-core-rust/src/lib.rs +1603 -0
- package/templates/icp-knowledge-canister/canisters/knolo-icp-canister/Cargo.lock +541 -0
- package/templates/icp-knowledge-canister/canisters/knolo-icp-canister/Cargo.toml +20 -0
- package/templates/icp-knowledge-canister/canisters/knolo-icp-canister/knolo_icp.did +29 -0
- package/templates/icp-knowledge-canister/canisters/knolo-icp-canister/src/lib.rs +605 -0
- package/templates/icp-knowledge-canister/dfx.json +25 -0
- package/templates/icp-knowledge-canister/frontend/index.html +12 -0
- package/templates/icp-knowledge-canister/frontend/package.json +27 -0
- package/templates/icp-knowledge-canister/frontend/src/App.tsx +225 -0
- package/templates/icp-knowledge-canister/frontend/src/app.css +271 -0
- package/templates/icp-knowledge-canister/frontend/src/canister.ts +119 -0
- package/templates/icp-knowledge-canister/frontend/src/canisterConfig.test.ts +34 -0
- package/templates/icp-knowledge-canister/frontend/src/canisterConfig.ts +26 -0
- package/templates/icp-knowledge-canister/frontend/src/main.tsx +5 -0
- package/templates/icp-knowledge-canister/frontend/src/vite-env.d.ts +14 -0
- package/templates/icp-knowledge-canister/frontend/tsconfig.json +18 -0
- package/templates/icp-knowledge-canister/frontend/vite.config.ts +66 -0
- package/templates/icp-knowledge-canister/knolo.config.json +15 -0
- package/templates/icp-knowledge-canister/knowledge/alpha.md +5 -0
- package/templates/icp-knowledge-canister/knowledge/guides/beta.md +5 -0
- package/templates/icp-knowledge-canister/scripts/build-sample-pack.mjs +24 -0
- package/templates/icp-knowledge-canister/scripts/query.sh +11 -0
- package/templates/icp-knowledge-canister/scripts/upload-pack.sh +16 -0
|
@@ -0,0 +1,1603 @@
|
|
|
1
|
+
use std::collections::{HashMap, HashSet};
|
|
2
|
+
use std::error::Error;
|
|
3
|
+
use std::fmt::{Display, Formatter};
|
|
4
|
+
|
|
5
|
+
#[derive(Debug, Clone)]
|
|
6
|
+
pub enum KnoloError {
|
|
7
|
+
InvalidPack(String),
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
impl Display for KnoloError {
|
|
11
|
+
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
|
|
12
|
+
match self {
|
|
13
|
+
KnoloError::InvalidPack(msg) => write!(f, "invalid pack: {msg}"),
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
impl Error for KnoloError {}
|
|
19
|
+
|
|
20
|
+
#[derive(Debug, Clone)]
|
|
21
|
+
pub struct PackMeta {
|
|
22
|
+
pub version: u32,
|
|
23
|
+
pub stats: PackStats,
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
#[derive(Debug, Clone)]
|
|
27
|
+
pub struct PackStats {
|
|
28
|
+
pub docs: usize,
|
|
29
|
+
pub blocks: usize,
|
|
30
|
+
pub terms: usize,
|
|
31
|
+
pub avg_block_len: Option<f64>,
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
#[derive(Debug, Clone)]
|
|
35
|
+
pub struct Pack {
|
|
36
|
+
pub meta: PackMeta,
|
|
37
|
+
pub lexicon: HashMap<String, u32>,
|
|
38
|
+
pub postings: Vec<u32>,
|
|
39
|
+
pub blocks: Vec<String>,
|
|
40
|
+
pub headings: Vec<Option<String>>,
|
|
41
|
+
pub doc_ids: Vec<Option<String>>,
|
|
42
|
+
pub namespaces: Vec<Option<String>>,
|
|
43
|
+
pub block_token_lens: Vec<usize>,
|
|
44
|
+
pub metadata_json: String,
|
|
45
|
+
pub claims_json: Option<String>,
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
#[derive(Debug, Clone)]
|
|
49
|
+
pub struct QueryOptions {
|
|
50
|
+
pub top_k: usize,
|
|
51
|
+
pub min_score: f64,
|
|
52
|
+
pub namespace: Option<Vec<String>>,
|
|
53
|
+
pub source: Option<Vec<String>>,
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
impl Default for QueryOptions {
|
|
57
|
+
fn default() -> Self {
|
|
58
|
+
Self {
|
|
59
|
+
top_k: 10,
|
|
60
|
+
min_score: 0.0,
|
|
61
|
+
namespace: None,
|
|
62
|
+
source: None,
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
#[derive(Debug, Clone, PartialEq)]
|
|
68
|
+
pub struct Hit {
|
|
69
|
+
pub block_id: usize,
|
|
70
|
+
pub score: f64,
|
|
71
|
+
pub text: String,
|
|
72
|
+
pub source: Option<String>,
|
|
73
|
+
pub namespace: Option<String>,
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
pub fn mount_pack_from_bytes(bytes: &[u8]) -> Result<Pack, KnoloError> {
|
|
77
|
+
let mut cursor = 0usize;
|
|
78
|
+
|
|
79
|
+
let meta_len = read_u32(bytes, &mut cursor)? as usize;
|
|
80
|
+
let meta_json = read_slice(bytes, &mut cursor, meta_len)?;
|
|
81
|
+
let metadata_json = std::str::from_utf8(meta_json).map_err(|_| KnoloError::InvalidPack("meta utf8".into()))?.to_string();
|
|
82
|
+
let meta = parse_meta(&metadata_json)?;
|
|
83
|
+
|
|
84
|
+
let lex_len = read_u32(bytes, &mut cursor)? as usize;
|
|
85
|
+
let lex_json = read_slice(bytes, &mut cursor, lex_len)?;
|
|
86
|
+
let lexicon = parse_lexicon(std::str::from_utf8(lex_json).map_err(|_| KnoloError::InvalidPack("lexicon utf8".into()))?)?;
|
|
87
|
+
|
|
88
|
+
let post_count = read_u32(bytes, &mut cursor)? as usize;
|
|
89
|
+
let postings = read_u32_array(bytes, &mut cursor, post_count)?;
|
|
90
|
+
|
|
91
|
+
let blocks_len = read_u32(bytes, &mut cursor)? as usize;
|
|
92
|
+
let blocks_json = read_slice(bytes, &mut cursor, blocks_len)?;
|
|
93
|
+
let blocks_str = std::str::from_utf8(blocks_json).map_err(|_| KnoloError::InvalidPack("blocks utf8".into()))?;
|
|
94
|
+
let parsed_blocks = parse_blocks(blocks_str)?;
|
|
95
|
+
|
|
96
|
+
Ok(Pack {
|
|
97
|
+
meta,
|
|
98
|
+
lexicon,
|
|
99
|
+
postings,
|
|
100
|
+
blocks: parsed_blocks.texts,
|
|
101
|
+
headings: parsed_blocks.headings,
|
|
102
|
+
doc_ids: parsed_blocks.doc_ids,
|
|
103
|
+
namespaces: parsed_blocks.namespaces,
|
|
104
|
+
block_token_lens: parsed_blocks.lens,
|
|
105
|
+
metadata_json,
|
|
106
|
+
claims_json: None,
|
|
107
|
+
})
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
pub fn query(pack: &Pack, q: &str, opts: QueryOptions) -> Vec<Hit> {
|
|
111
|
+
if q.trim().is_empty() {
|
|
112
|
+
return vec![];
|
|
113
|
+
}
|
|
114
|
+
let tokens = tokenize(q);
|
|
115
|
+
if tokens.is_empty() {
|
|
116
|
+
return vec![];
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
let term_ids = tokens
|
|
120
|
+
.iter()
|
|
121
|
+
.filter_map(|t| pack.lexicon.get(t).copied())
|
|
122
|
+
.collect::<HashSet<_>>();
|
|
123
|
+
if term_ids.is_empty() {
|
|
124
|
+
return vec![];
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
let namespace_filter = normalize_filter(opts.namespace.as_ref());
|
|
128
|
+
let source_filter = normalize_filter(opts.source.as_ref());
|
|
129
|
+
|
|
130
|
+
let mut candidates: HashMap<usize, HashMap<u32, f64>> = HashMap::new();
|
|
131
|
+
let mut dfs: HashMap<u32, usize> = HashMap::new();
|
|
132
|
+
let uses_offset_block_ids = pack.meta.version >= 3;
|
|
133
|
+
|
|
134
|
+
let mut i = 0usize;
|
|
135
|
+
while i < pack.postings.len() {
|
|
136
|
+
let tid = pack.postings[i];
|
|
137
|
+
i += 1;
|
|
138
|
+
if tid == 0 {
|
|
139
|
+
continue;
|
|
140
|
+
}
|
|
141
|
+
let relevant = term_ids.contains(&tid);
|
|
142
|
+
let mut term_df = 0usize;
|
|
143
|
+
|
|
144
|
+
if i >= pack.postings.len() { break; }
|
|
145
|
+
let mut encoded_bid = pack.postings[i];
|
|
146
|
+
i += 1;
|
|
147
|
+
|
|
148
|
+
while encoded_bid != 0 && i < pack.postings.len() {
|
|
149
|
+
let bid = if uses_offset_block_ids {
|
|
150
|
+
encoded_bid.saturating_sub(1) as usize
|
|
151
|
+
} else {
|
|
152
|
+
encoded_bid as usize
|
|
153
|
+
};
|
|
154
|
+
|
|
155
|
+
let mut tf = 0usize;
|
|
156
|
+
while i < pack.postings.len() {
|
|
157
|
+
let pos = pack.postings[i];
|
|
158
|
+
i += 1;
|
|
159
|
+
if pos == 0 {
|
|
160
|
+
break;
|
|
161
|
+
}
|
|
162
|
+
tf += 1;
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
term_df += 1;
|
|
166
|
+
if relevant && bid < pack.blocks.len() {
|
|
167
|
+
let entry = candidates.entry(bid).or_default();
|
|
168
|
+
*entry.entry(tid).or_insert(0.0) += tf as f64;
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
if i >= pack.postings.len() { break; }
|
|
172
|
+
encoded_bid = pack.postings[i];
|
|
173
|
+
i += 1;
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
if relevant {
|
|
177
|
+
dfs.insert(tid, term_df);
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
if !namespace_filter.is_empty() {
|
|
182
|
+
candidates.retain(|bid, _| {
|
|
183
|
+
pack.namespaces
|
|
184
|
+
.get(*bid)
|
|
185
|
+
.and_then(|n| n.clone())
|
|
186
|
+
.map(|n| namespace_filter.contains(&normalize(&n)))
|
|
187
|
+
.unwrap_or(false)
|
|
188
|
+
});
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
if !source_filter.is_empty() {
|
|
192
|
+
candidates.retain(|bid, _| {
|
|
193
|
+
pack.doc_ids
|
|
194
|
+
.get(*bid)
|
|
195
|
+
.and_then(|n| n.clone())
|
|
196
|
+
.map(|n| source_filter.contains(&normalize(&n)))
|
|
197
|
+
.unwrap_or(false)
|
|
198
|
+
});
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
let doc_count = pack.meta.stats.blocks.max(1) as f64;
|
|
202
|
+
let avg_len = pack
|
|
203
|
+
.meta
|
|
204
|
+
.stats
|
|
205
|
+
.avg_block_len
|
|
206
|
+
.unwrap_or_else(|| {
|
|
207
|
+
if pack.block_token_lens.is_empty() {
|
|
208
|
+
1.0
|
|
209
|
+
} else {
|
|
210
|
+
pack.block_token_lens.iter().sum::<usize>() as f64 / pack.block_token_lens.len() as f64
|
|
211
|
+
}
|
|
212
|
+
})
|
|
213
|
+
.max(1.0);
|
|
214
|
+
|
|
215
|
+
let mut scored = candidates
|
|
216
|
+
.into_iter()
|
|
217
|
+
.map(|(bid, tf_map)| {
|
|
218
|
+
let mut score = 0.0;
|
|
219
|
+
let len = *pack.block_token_lens.get(bid).unwrap_or(&1) as f64;
|
|
220
|
+
for (tid, tf) in tf_map {
|
|
221
|
+
let df = *dfs.get(&tid).unwrap_or(&0) as f64;
|
|
222
|
+
let idf = (1.0 + (doc_count - df + 0.5) / (df + 0.5)).ln();
|
|
223
|
+
let k1 = 1.5;
|
|
224
|
+
let b = 0.75;
|
|
225
|
+
let numer = tf * (k1 + 1.0);
|
|
226
|
+
let denom = tf + k1 * (1.0 - b + b * (len / avg_len));
|
|
227
|
+
score += idf * (numer / denom);
|
|
228
|
+
}
|
|
229
|
+
(bid, score)
|
|
230
|
+
})
|
|
231
|
+
.filter(|(_, score)| *score >= opts.min_score)
|
|
232
|
+
.collect::<Vec<_>>();
|
|
233
|
+
|
|
234
|
+
scored.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));
|
|
235
|
+
|
|
236
|
+
scored
|
|
237
|
+
.into_iter()
|
|
238
|
+
.take(opts.top_k.max(1))
|
|
239
|
+
.map(|(bid, score)| Hit {
|
|
240
|
+
block_id: bid,
|
|
241
|
+
score,
|
|
242
|
+
text: pack.blocks.get(bid).cloned().unwrap_or_default(),
|
|
243
|
+
source: pack.doc_ids.get(bid).and_then(|s| s.clone()),
|
|
244
|
+
namespace: pack.namespaces.get(bid).and_then(|s| s.clone()),
|
|
245
|
+
})
|
|
246
|
+
.collect()
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
struct ParsedBlocks {
|
|
250
|
+
texts: Vec<String>,
|
|
251
|
+
headings: Vec<Option<String>>,
|
|
252
|
+
doc_ids: Vec<Option<String>>,
|
|
253
|
+
namespaces: Vec<Option<String>>,
|
|
254
|
+
lens: Vec<usize>,
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
fn parse_meta(json: &str) -> Result<PackMeta, KnoloError> {
|
|
258
|
+
Ok(PackMeta {
|
|
259
|
+
version: parse_u32_field(json, "version")?,
|
|
260
|
+
stats: PackStats {
|
|
261
|
+
docs: parse_u32_field(json, "docs")? as usize,
|
|
262
|
+
blocks: parse_u32_field(json, "blocks")? as usize,
|
|
263
|
+
terms: parse_u32_field(json, "terms")? as usize,
|
|
264
|
+
avg_block_len: parse_f64_field(json, "avgBlockLen"),
|
|
265
|
+
},
|
|
266
|
+
})
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
fn parse_lexicon(json: &str) -> Result<HashMap<String, u32>, KnoloError> {
|
|
270
|
+
let mut map = HashMap::new();
|
|
271
|
+
let s = compact(json);
|
|
272
|
+
let mut i = 0usize;
|
|
273
|
+
while let Some(start) = s[i..].find("[\"") {
|
|
274
|
+
let abs = i + start + 2;
|
|
275
|
+
let rest = &s[abs..];
|
|
276
|
+
let end = rest.find('"').ok_or_else(|| KnoloError::InvalidPack("lexicon key".into()))?;
|
|
277
|
+
let key = rest[..end].to_string();
|
|
278
|
+
let rest2 = &rest[end + 1..];
|
|
279
|
+
let comma = rest2.find(',').ok_or_else(|| KnoloError::InvalidPack("lexicon comma".into()))?;
|
|
280
|
+
let rest3 = &rest2[comma + 1..];
|
|
281
|
+
let mut n = String::new();
|
|
282
|
+
for ch in rest3.chars() {
|
|
283
|
+
if ch.is_ascii_digit() {
|
|
284
|
+
n.push(ch);
|
|
285
|
+
} else {
|
|
286
|
+
break;
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
if !n.is_empty() {
|
|
290
|
+
map.insert(key, n.parse::<u32>().map_err(|_| KnoloError::InvalidPack("lexicon tid".into()))?);
|
|
291
|
+
}
|
|
292
|
+
i = abs + end + 1;
|
|
293
|
+
}
|
|
294
|
+
Ok(map)
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
fn parse_blocks(json: &str) -> Result<ParsedBlocks, KnoloError> {
|
|
298
|
+
let s = compact(json);
|
|
299
|
+
if s.starts_with("[\"") {
|
|
300
|
+
let mut texts = Vec::new();
|
|
301
|
+
let mut i = 2usize;
|
|
302
|
+
while i < s.len() {
|
|
303
|
+
if let Some(end) = s[i..].find('"') {
|
|
304
|
+
let piece = &s[i..i + end];
|
|
305
|
+
texts.push(unescape(piece));
|
|
306
|
+
i += end + 1;
|
|
307
|
+
if let Some(next) = s[i..].find('"') {
|
|
308
|
+
i += next + 1;
|
|
309
|
+
} else {
|
|
310
|
+
break;
|
|
311
|
+
}
|
|
312
|
+
} else {
|
|
313
|
+
break;
|
|
314
|
+
}
|
|
315
|
+
}
|
|
316
|
+
let lens = texts.iter().map(|t| tokenize(t).len()).collect::<Vec<_>>();
|
|
317
|
+
return Ok(ParsedBlocks {
|
|
318
|
+
headings: vec![None; texts.len()],
|
|
319
|
+
doc_ids: vec![None; texts.len()],
|
|
320
|
+
namespaces: vec![None; texts.len()],
|
|
321
|
+
lens,
|
|
322
|
+
texts,
|
|
323
|
+
});
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
let objects = split_top_level_objects(&s)?;
|
|
327
|
+
let mut texts = Vec::new();
|
|
328
|
+
let mut headings = Vec::new();
|
|
329
|
+
let mut doc_ids = Vec::new();
|
|
330
|
+
let mut namespaces = Vec::new();
|
|
331
|
+
let mut lens = Vec::new();
|
|
332
|
+
|
|
333
|
+
for obj in objects {
|
|
334
|
+
let text = parse_string_or_null(&obj, "text").unwrap_or_default();
|
|
335
|
+
let len = parse_u32_field_optional(&obj, "len").map(|v| v as usize).unwrap_or_else(|| tokenize(&text).len());
|
|
336
|
+
texts.push(text);
|
|
337
|
+
headings.push(parse_string_or_null(&obj, "heading"));
|
|
338
|
+
doc_ids.push(parse_string_or_null(&obj, "docId"));
|
|
339
|
+
namespaces.push(parse_string_or_null(&obj, "namespace"));
|
|
340
|
+
lens.push(len);
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
Ok(ParsedBlocks { texts, headings, doc_ids, namespaces, lens })
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
fn split_top_level_objects(s: &str) -> Result<Vec<String>, KnoloError> {
|
|
347
|
+
let mut out = Vec::new();
|
|
348
|
+
let mut depth = 0i32;
|
|
349
|
+
let mut start = None;
|
|
350
|
+
let chars: Vec<char> = s.chars().collect();
|
|
351
|
+
for (i, ch) in chars.iter().enumerate() {
|
|
352
|
+
if *ch == '{' {
|
|
353
|
+
if depth == 0 {
|
|
354
|
+
start = Some(i);
|
|
355
|
+
}
|
|
356
|
+
depth += 1;
|
|
357
|
+
} else if *ch == '}' {
|
|
358
|
+
depth -= 1;
|
|
359
|
+
if depth == 0 {
|
|
360
|
+
if let Some(st) = start {
|
|
361
|
+
out.push(chars[st..=i].iter().collect());
|
|
362
|
+
}
|
|
363
|
+
start = None;
|
|
364
|
+
}
|
|
365
|
+
}
|
|
366
|
+
}
|
|
367
|
+
if out.is_empty() {
|
|
368
|
+
return Err(KnoloError::InvalidPack("blocks objects".into()));
|
|
369
|
+
}
|
|
370
|
+
Ok(out)
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
fn parse_string_or_null(obj: &str, key: &str) -> Option<String> {
|
|
374
|
+
let needle = format!("\"{}\":", key);
|
|
375
|
+
let idx = obj.find(&needle)? + needle.len();
|
|
376
|
+
let tail = &obj[idx..];
|
|
377
|
+
if tail.starts_with("null") {
|
|
378
|
+
return None;
|
|
379
|
+
}
|
|
380
|
+
if !tail.starts_with('"') {
|
|
381
|
+
return None;
|
|
382
|
+
}
|
|
383
|
+
let rest = &tail[1..];
|
|
384
|
+
let end = rest.find('"')?;
|
|
385
|
+
Some(unescape(&rest[..end]))
|
|
386
|
+
}
|
|
387
|
+
|
|
388
|
+
fn parse_u32_field(json: &str, key: &str) -> Result<u32, KnoloError> {
|
|
389
|
+
parse_u32_field_optional(json, key).ok_or_else(|| KnoloError::InvalidPack(format!("missing {key}")))
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
fn parse_u32_field_optional(json: &str, key: &str) -> Option<u32> {
|
|
393
|
+
let needle = format!("\"{}\":", key);
|
|
394
|
+
let idx = json.find(&needle)? + needle.len();
|
|
395
|
+
let tail = &json[idx..];
|
|
396
|
+
let mut n = String::new();
|
|
397
|
+
for ch in tail.chars() {
|
|
398
|
+
if ch.is_ascii_digit() {
|
|
399
|
+
n.push(ch);
|
|
400
|
+
} else if !n.is_empty() {
|
|
401
|
+
break;
|
|
402
|
+
}
|
|
403
|
+
}
|
|
404
|
+
n.parse().ok()
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
fn parse_f64_field(json: &str, key: &str) -> Option<f64> {
|
|
408
|
+
let needle = format!("\"{}\":", key);
|
|
409
|
+
let idx = json.find(&needle)? + needle.len();
|
|
410
|
+
let tail = &json[idx..];
|
|
411
|
+
let mut n = String::new();
|
|
412
|
+
for ch in tail.chars() {
|
|
413
|
+
if ch.is_ascii_digit() || ch == '.' {
|
|
414
|
+
n.push(ch);
|
|
415
|
+
} else if !n.is_empty() {
|
|
416
|
+
break;
|
|
417
|
+
}
|
|
418
|
+
}
|
|
419
|
+
n.parse().ok()
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
fn normalize_filter(values: Option<&Vec<String>>) -> HashSet<String> {
|
|
423
|
+
values
|
|
424
|
+
.map(|arr| arr.iter().map(|s| normalize(s)).collect::<HashSet<_>>())
|
|
425
|
+
.unwrap_or_default()
|
|
426
|
+
}
|
|
427
|
+
|
|
428
|
+
fn normalize(s: &str) -> String {
|
|
429
|
+
s.to_lowercase().trim().to_string()
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
fn tokenize(text: &str) -> Vec<String> {
|
|
433
|
+
let mut out = Vec::new();
|
|
434
|
+
let mut cur = String::new();
|
|
435
|
+
for ch in text.chars() {
|
|
436
|
+
if ch.is_alphanumeric() {
|
|
437
|
+
cur.push(ch.to_ascii_lowercase());
|
|
438
|
+
} else if !cur.is_empty() {
|
|
439
|
+
out.push(std::mem::take(&mut cur));
|
|
440
|
+
}
|
|
441
|
+
}
|
|
442
|
+
if !cur.is_empty() {
|
|
443
|
+
out.push(cur);
|
|
444
|
+
}
|
|
445
|
+
out
|
|
446
|
+
}
|
|
447
|
+
|
|
448
|
+
fn compact(s: &str) -> String {
|
|
449
|
+
let mut out = String::with_capacity(s.len());
|
|
450
|
+
let mut in_string = false;
|
|
451
|
+
let mut escaped = false;
|
|
452
|
+
|
|
453
|
+
for ch in s.chars() {
|
|
454
|
+
if in_string {
|
|
455
|
+
out.push(ch);
|
|
456
|
+
if escaped {
|
|
457
|
+
escaped = false;
|
|
458
|
+
} else if ch == '\\' {
|
|
459
|
+
escaped = true;
|
|
460
|
+
} else if ch == '"' {
|
|
461
|
+
in_string = false;
|
|
462
|
+
}
|
|
463
|
+
continue;
|
|
464
|
+
}
|
|
465
|
+
|
|
466
|
+
if ch.is_whitespace() {
|
|
467
|
+
continue;
|
|
468
|
+
}
|
|
469
|
+
|
|
470
|
+
out.push(ch);
|
|
471
|
+
if ch == '"' {
|
|
472
|
+
in_string = true;
|
|
473
|
+
}
|
|
474
|
+
}
|
|
475
|
+
|
|
476
|
+
out
|
|
477
|
+
}
|
|
478
|
+
|
|
479
|
+
fn unescape(s: &str) -> String {
|
|
480
|
+
s.replace("\\\"", "\"")
|
|
481
|
+
}
|
|
482
|
+
|
|
483
|
+
fn read_u32(bytes: &[u8], cursor: &mut usize) -> Result<u32, KnoloError> {
|
|
484
|
+
let chunk = read_slice(bytes, cursor, 4)?;
|
|
485
|
+
Ok(u32::from_le_bytes([chunk[0], chunk[1], chunk[2], chunk[3]]))
|
|
486
|
+
}
|
|
487
|
+
|
|
488
|
+
fn read_u32_array(bytes: &[u8], cursor: &mut usize, len: usize) -> Result<Vec<u32>, KnoloError> {
|
|
489
|
+
let mut out = Vec::with_capacity(len);
|
|
490
|
+
for _ in 0..len {
|
|
491
|
+
out.push(read_u32(bytes, cursor)?);
|
|
492
|
+
}
|
|
493
|
+
Ok(out)
|
|
494
|
+
}
|
|
495
|
+
|
|
496
|
+
fn read_slice<'a>(bytes: &'a [u8], cursor: &mut usize, len: usize) -> Result<&'a [u8], KnoloError> {
|
|
497
|
+
let end = cursor.saturating_add(len);
|
|
498
|
+
if end > bytes.len() {
|
|
499
|
+
return Err(KnoloError::InvalidPack("unexpected end-of-buffer".into()));
|
|
500
|
+
}
|
|
501
|
+
let slice = &bytes[*cursor..end];
|
|
502
|
+
*cursor = end;
|
|
503
|
+
Ok(slice)
|
|
504
|
+
}
|
|
505
|
+
|
|
506
|
+
// V5 read-only Knowledge Image support. The implementation intentionally has
|
|
507
|
+
// no external dependencies so the format verifier can be used in offline and
|
|
508
|
+
// embedded environments.
|
|
509
|
+
|
|
510
|
+
pub const KNOWLEDGE_IMAGE_V5_MAGIC: &[u8; 8] = b"KNLOV5\0\0";
|
|
511
|
+
const SUPERBLOCK_MAGIC: &[u8; 8] = b"KNLOSB1\0";
|
|
512
|
+
const SEGMENT_MAGIC: &[u8; 4] = b"KSEG";
|
|
513
|
+
const V5_HEADER_SIZE: usize = 16;
|
|
514
|
+
const V5_SUPERBLOCK_SIZE: usize = 128;
|
|
515
|
+
const V5_SEGMENT_HEADER_SIZE: usize = 48;
|
|
516
|
+
const V5_DATA_START: usize = V5_HEADER_SIZE + V5_SUPERBLOCK_SIZE * 2;
|
|
517
|
+
const V5_MAX_SEGMENT: u64 = 512 * 1024 * 1024;
|
|
518
|
+
|
|
519
|
+
#[derive(Debug, Clone, PartialEq)]
|
|
520
|
+
pub struct KnowledgeObjectV1 {
|
|
521
|
+
pub id: String,
|
|
522
|
+
pub kind: String,
|
|
523
|
+
pub bytes: Vec<u8>,
|
|
524
|
+
pub meta: CborValue,
|
|
525
|
+
}
|
|
526
|
+
|
|
527
|
+
#[derive(Debug, Clone, PartialEq)]
|
|
528
|
+
pub struct KnowledgeEventV1 {
|
|
529
|
+
pub id: String,
|
|
530
|
+
pub transaction_id: String,
|
|
531
|
+
pub actor: String,
|
|
532
|
+
pub actor_counter: u64,
|
|
533
|
+
pub kind: String,
|
|
534
|
+
pub target: String,
|
|
535
|
+
pub payload: String,
|
|
536
|
+
}
|
|
537
|
+
|
|
538
|
+
#[derive(Debug, Clone, PartialEq)]
|
|
539
|
+
pub struct KnowledgeCommitV1 {
|
|
540
|
+
pub state_root: String,
|
|
541
|
+
pub commit_digest: String,
|
|
542
|
+
pub parents: Vec<String>,
|
|
543
|
+
pub object_root: String,
|
|
544
|
+
pub event_root: String,
|
|
545
|
+
pub policy_root: String,
|
|
546
|
+
pub sequence: u64,
|
|
547
|
+
pub actor: String,
|
|
548
|
+
}
|
|
549
|
+
|
|
550
|
+
#[derive(Debug, Clone, PartialEq)]
|
|
551
|
+
pub struct KnowledgeImageSegment {
|
|
552
|
+
pub kind: u8,
|
|
553
|
+
pub schema: u8,
|
|
554
|
+
pub flags: u16,
|
|
555
|
+
pub offset: usize,
|
|
556
|
+
pub length: usize,
|
|
557
|
+
pub payload_length: usize,
|
|
558
|
+
pub digest: String,
|
|
559
|
+
}
|
|
560
|
+
|
|
561
|
+
#[derive(Debug, Clone, PartialEq)]
|
|
562
|
+
pub struct KnowledgeImage {
|
|
563
|
+
pub state_root: String,
|
|
564
|
+
pub commit_digest: String,
|
|
565
|
+
pub commit: KnowledgeCommitV1,
|
|
566
|
+
pub objects: Vec<KnowledgeObjectV1>,
|
|
567
|
+
pub events: Vec<KnowledgeEventV1>,
|
|
568
|
+
pub segments: Vec<KnowledgeImageSegment>,
|
|
569
|
+
pub active_superblock: char,
|
|
570
|
+
}
|
|
571
|
+
|
|
572
|
+
#[derive(Debug, Clone, PartialEq)]
|
|
573
|
+
pub struct KnowledgeImageVerification {
|
|
574
|
+
pub valid: bool,
|
|
575
|
+
pub state_root: String,
|
|
576
|
+
pub commit_digest: String,
|
|
577
|
+
pub active_superblock: char,
|
|
578
|
+
pub segments: Vec<KnowledgeImageSegment>,
|
|
579
|
+
}
|
|
580
|
+
|
|
581
|
+
#[derive(Debug, Clone, PartialEq)]
|
|
582
|
+
pub struct MigrationMapping {
|
|
583
|
+
pub legacy_block_id: usize,
|
|
584
|
+
pub source_object: String,
|
|
585
|
+
pub chunk_object: String,
|
|
586
|
+
}
|
|
587
|
+
|
|
588
|
+
#[derive(Debug, Clone, PartialEq)]
|
|
589
|
+
pub struct MigrationResult {
|
|
590
|
+
pub image: Vec<u8>,
|
|
591
|
+
pub receipt: Vec<u8>,
|
|
592
|
+
pub state_root: String,
|
|
593
|
+
pub mappings: Vec<MigrationMapping>,
|
|
594
|
+
}
|
|
595
|
+
|
|
596
|
+
#[derive(Debug, Clone, PartialEq)]
|
|
597
|
+
pub enum CborValue {
|
|
598
|
+
Null,
|
|
599
|
+
Bool(bool),
|
|
600
|
+
UInt(u64),
|
|
601
|
+
NInt(i64),
|
|
602
|
+
Bytes(Vec<u8>),
|
|
603
|
+
Text(String),
|
|
604
|
+
Array(Vec<CborValue>),
|
|
605
|
+
Map(Vec<(String, CborValue)>),
|
|
606
|
+
}
|
|
607
|
+
|
|
608
|
+
pub fn mount_knowledge_image(bytes: &[u8]) -> Result<KnowledgeImage, KnoloError> {
|
|
609
|
+
let parsed = parse_v5_image(bytes)?;
|
|
610
|
+
Ok(parsed)
|
|
611
|
+
}
|
|
612
|
+
|
|
613
|
+
pub fn inspect_knowledge_image(bytes: &[u8]) -> Result<KnowledgeImageVerification, KnoloError> {
|
|
614
|
+
let image = mount_knowledge_image(bytes)?;
|
|
615
|
+
Ok(KnowledgeImageVerification {
|
|
616
|
+
valid: true,
|
|
617
|
+
state_root: image.state_root,
|
|
618
|
+
commit_digest: image.commit_digest,
|
|
619
|
+
active_superblock: image.active_superblock,
|
|
620
|
+
segments: image.segments,
|
|
621
|
+
})
|
|
622
|
+
}
|
|
623
|
+
|
|
624
|
+
pub fn verify_knowledge_image(bytes: &[u8]) -> Result<KnowledgeImageVerification, KnoloError> {
|
|
625
|
+
inspect_knowledge_image(bytes)
|
|
626
|
+
}
|
|
627
|
+
|
|
628
|
+
pub fn state_root(image: &KnowledgeImage) -> &str {
|
|
629
|
+
&image.state_root
|
|
630
|
+
}
|
|
631
|
+
|
|
632
|
+
#[derive(Debug, Clone, PartialEq)]
|
|
633
|
+
pub struct KnowledgeQueryFilterV1 {
|
|
634
|
+
pub field: String,
|
|
635
|
+
pub value: CborValue,
|
|
636
|
+
}
|
|
637
|
+
|
|
638
|
+
#[derive(Debug, Clone, PartialEq)]
|
|
639
|
+
pub struct KnowledgeQueryPlanV1 {
|
|
640
|
+
pub kind: Option<String>,
|
|
641
|
+
pub filters: Vec<KnowledgeQueryFilterV1>,
|
|
642
|
+
pub search: Option<String>,
|
|
643
|
+
pub limit: usize,
|
|
644
|
+
}
|
|
645
|
+
|
|
646
|
+
#[derive(Debug, Clone, PartialEq)]
|
|
647
|
+
pub struct KnowledgeQueryHitV1 {
|
|
648
|
+
pub object_id: String,
|
|
649
|
+
pub kind: String,
|
|
650
|
+
}
|
|
651
|
+
|
|
652
|
+
#[derive(Debug, Clone, PartialEq)]
|
|
653
|
+
pub struct KnowledgeQueryResultV1 {
|
|
654
|
+
pub state_root: String,
|
|
655
|
+
pub plan_root: String,
|
|
656
|
+
pub hits: Vec<KnowledgeQueryHitV1>,
|
|
657
|
+
pub result_root: String,
|
|
658
|
+
}
|
|
659
|
+
|
|
660
|
+
#[derive(Debug, Clone, PartialEq)]
|
|
661
|
+
pub struct KnowledgePolicyRuleV1 {
|
|
662
|
+
pub effect: String,
|
|
663
|
+
pub action: String,
|
|
664
|
+
pub principal: Option<String>,
|
|
665
|
+
pub kind: Option<String>,
|
|
666
|
+
}
|
|
667
|
+
|
|
668
|
+
#[derive(Debug, Clone, PartialEq)]
|
|
669
|
+
pub struct KnowledgePolicyV1 {
|
|
670
|
+
pub default: String,
|
|
671
|
+
pub rules: Vec<KnowledgePolicyRuleV1>,
|
|
672
|
+
}
|
|
673
|
+
|
|
674
|
+
#[derive(Debug, Clone, PartialEq)]
|
|
675
|
+
pub struct KnowledgeAuthorizationResultV1 {
|
|
676
|
+
pub state_root: String,
|
|
677
|
+
pub plan_root: String,
|
|
678
|
+
pub policy_root: String,
|
|
679
|
+
pub authorization_root: String,
|
|
680
|
+
pub principal: String,
|
|
681
|
+
pub action: String,
|
|
682
|
+
pub decision: String,
|
|
683
|
+
pub allowed_object_ids: Vec<String>,
|
|
684
|
+
pub denied_object_ids: Vec<String>,
|
|
685
|
+
}
|
|
686
|
+
|
|
687
|
+
#[derive(Debug, Clone, PartialEq)]
|
|
688
|
+
pub struct KnowledgeDelegationV1 {
|
|
689
|
+
pub version: u64,
|
|
690
|
+
pub delegator: String,
|
|
691
|
+
pub delegatee: String,
|
|
692
|
+
pub action: String,
|
|
693
|
+
pub issued_at: u64,
|
|
694
|
+
pub expires_at: u64,
|
|
695
|
+
pub algorithm: String,
|
|
696
|
+
pub key_id: Option<String>,
|
|
697
|
+
pub signature: Vec<u8>,
|
|
698
|
+
}
|
|
699
|
+
|
|
700
|
+
#[derive(Debug, Clone, PartialEq)]
|
|
701
|
+
pub struct KnowledgeAuthorityEnvelopeV1 {
|
|
702
|
+
pub version: u64,
|
|
703
|
+
pub issuer: String,
|
|
704
|
+
pub subject: String,
|
|
705
|
+
pub authorization_root: String,
|
|
706
|
+
pub keyring_root: Option<String>,
|
|
707
|
+
pub issued_at: u64,
|
|
708
|
+
pub expires_at: u64,
|
|
709
|
+
pub algorithm: String,
|
|
710
|
+
pub key_id: Option<String>,
|
|
711
|
+
pub delegations: Vec<KnowledgeDelegationV1>,
|
|
712
|
+
pub signature: Vec<u8>,
|
|
713
|
+
}
|
|
714
|
+
|
|
715
|
+
#[derive(Debug, Clone, PartialEq)]
|
|
716
|
+
pub struct KnowledgeAuthorityVerificationV1 {
|
|
717
|
+
pub envelope_root: String,
|
|
718
|
+
pub issuer: String,
|
|
719
|
+
pub subject: String,
|
|
720
|
+
pub authorization_root: String,
|
|
721
|
+
pub keyring_root: Option<String>,
|
|
722
|
+
pub delegation_depth: usize,
|
|
723
|
+
}
|
|
724
|
+
|
|
725
|
+
pub fn knowledge_policy_root_v5(policy: &KnowledgePolicyV1) -> Result<String, KnoloError> {
|
|
726
|
+
if policy.default != "allow" && policy.default != "deny" { return Err(KnoloError::InvalidPack("invalid V5 policy default".into())); }
|
|
727
|
+
let mut rules = policy.rules.clone();
|
|
728
|
+
for rule in &rules {
|
|
729
|
+
if !matches!(rule.effect.as_str(), "allow" | "deny") || !matches!(rule.action.as_str(), "query" | "read") { return Err(KnoloError::InvalidPack("invalid V5 policy rule".into())); }
|
|
730
|
+
if rule.principal.as_deref() == Some("") || rule.kind.as_deref() == Some("") { return Err(KnoloError::InvalidPack("invalid V5 policy selector".into())); }
|
|
731
|
+
}
|
|
732
|
+
rules.sort_by(|left, right| policy_rule_key(left).cmp(&policy_rule_key(right)));
|
|
733
|
+
let body = if rules.is_empty() {
|
|
734
|
+
CborValue::Map(vec![("default".into(), CborValue::Text(policy.default.clone()))])
|
|
735
|
+
} else {
|
|
736
|
+
CborValue::Map(vec![
|
|
737
|
+
("default".into(), CborValue::Text(policy.default.clone())),
|
|
738
|
+
("rules".into(), CborValue::Array(rules.iter().map(policy_rule_value).collect())),
|
|
739
|
+
])
|
|
740
|
+
};
|
|
741
|
+
Ok(digest_domain("policy", &encode_cbor(&body)))
|
|
742
|
+
}
|
|
743
|
+
|
|
744
|
+
pub fn evaluate_knowledge_query_policy_v5(
|
|
745
|
+
image: &KnowledgeImage,
|
|
746
|
+
query_result: &KnowledgeQueryResultV1,
|
|
747
|
+
policy: &KnowledgePolicyV1,
|
|
748
|
+
principal: &str,
|
|
749
|
+
action: &str,
|
|
750
|
+
) -> Result<KnowledgeAuthorizationResultV1, KnoloError> {
|
|
751
|
+
if principal.is_empty() || !matches!(action, "query" | "read") { return Err(KnoloError::InvalidPack("invalid V5 authorization input".into())); }
|
|
752
|
+
if query_result.state_root != image.state_root { return Err(KnoloError::InvalidPack("V5 query state root mismatch".into())); }
|
|
753
|
+
let policy_root = knowledge_policy_root_v5(policy)?;
|
|
754
|
+
if policy_root != image.commit.policy_root { return Err(KnoloError::InvalidPack("V5 policy root mismatch".into())); }
|
|
755
|
+
let mut allowed_object_ids = Vec::new();
|
|
756
|
+
let mut denied_object_ids = Vec::new();
|
|
757
|
+
for hit in &query_result.hits {
|
|
758
|
+
if authorize_policy_hit(hit, policy, principal, action) { allowed_object_ids.push(hit.object_id.clone()); } else { denied_object_ids.push(hit.object_id.clone()); }
|
|
759
|
+
}
|
|
760
|
+
let decision = if denied_object_ids.is_empty() { "allow" } else if allowed_object_ids.is_empty() { "deny" } else { "partial" };
|
|
761
|
+
let authorization_value = CborValue::Map(vec![
|
|
762
|
+
("action".into(), CborValue::Text(action.into())),
|
|
763
|
+
("allowedObjectIds".into(), CborValue::Array(allowed_object_ids.iter().map(|id| CborValue::Text(id.clone())).collect())),
|
|
764
|
+
("decision".into(), CborValue::Text(decision.into())),
|
|
765
|
+
("deniedObjectIds".into(), CborValue::Array(denied_object_ids.iter().map(|id| CborValue::Text(id.clone())).collect())),
|
|
766
|
+
("planRoot".into(), CborValue::Text(query_result.plan_root.clone())),
|
|
767
|
+
("policyRoot".into(), CborValue::Text(policy_root.clone())),
|
|
768
|
+
("principal".into(), CborValue::Text(principal.into())),
|
|
769
|
+
("stateRoot".into(), CborValue::Text(image.state_root.clone())),
|
|
770
|
+
]);
|
|
771
|
+
let authorization_root = digest_domain("authorization", &encode_cbor(&authorization_value));
|
|
772
|
+
Ok(KnowledgeAuthorizationResultV1 { state_root: image.state_root.clone(), plan_root: query_result.plan_root.clone(), policy_root, authorization_root, principal: principal.into(), action: action.into(), decision: decision.into(), allowed_object_ids, denied_object_ids })
|
|
773
|
+
}
|
|
774
|
+
|
|
775
|
+
pub fn delegation_payload_v1(delegation: &KnowledgeDelegationV1) -> Vec<u8> {
|
|
776
|
+
let mut entries = vec![
|
|
777
|
+
("action".into(), CborValue::Text(delegation.action.clone())),
|
|
778
|
+
("algorithm".into(), CborValue::Text(delegation.algorithm.clone())),
|
|
779
|
+
("delegatee".into(), CborValue::Text(delegation.delegatee.clone())),
|
|
780
|
+
("delegator".into(), CborValue::Text(delegation.delegator.clone())),
|
|
781
|
+
("expiresAt".into(), CborValue::UInt(delegation.expires_at)),
|
|
782
|
+
("issuedAt".into(), CborValue::UInt(delegation.issued_at)),
|
|
783
|
+
("version".into(), CborValue::UInt(delegation.version)),
|
|
784
|
+
];
|
|
785
|
+
if let Some(key_id) = &delegation.key_id { entries.push(("keyId".into(), CborValue::Text(key_id.clone()))); }
|
|
786
|
+
encode_cbor(&CborValue::Map(entries))
|
|
787
|
+
}
|
|
788
|
+
|
|
789
|
+
pub fn authority_envelope_payload_v1(envelope: &KnowledgeAuthorityEnvelopeV1) -> Vec<u8> {
|
|
790
|
+
let delegation_roots = envelope.delegations.iter().map(|delegation| digest_domain("delegation-payload", &delegation_payload_v1(delegation))).map(CborValue::Text).collect();
|
|
791
|
+
let mut entries = vec![
|
|
792
|
+
("algorithm".into(), CborValue::Text(envelope.algorithm.clone())),
|
|
793
|
+
("authorizationRoot".into(), CborValue::Text(envelope.authorization_root.clone())),
|
|
794
|
+
("delegations".into(), CborValue::Array(delegation_roots)),
|
|
795
|
+
("expiresAt".into(), CborValue::UInt(envelope.expires_at)),
|
|
796
|
+
("issuedAt".into(), CborValue::UInt(envelope.issued_at)),
|
|
797
|
+
("issuer".into(), CborValue::Text(envelope.issuer.clone())),
|
|
798
|
+
("subject".into(), CborValue::Text(envelope.subject.clone())),
|
|
799
|
+
("version".into(), CborValue::UInt(envelope.version)),
|
|
800
|
+
];
|
|
801
|
+
if let Some(keyring_root) = &envelope.keyring_root { entries.push(("keyringRoot".into(), CborValue::Text(keyring_root.clone()))); }
|
|
802
|
+
if let Some(key_id) = &envelope.key_id { entries.push(("keyId".into(), CborValue::Text(key_id.clone()))); }
|
|
803
|
+
encode_cbor(&CborValue::Map(entries))
|
|
804
|
+
}
|
|
805
|
+
|
|
806
|
+
pub fn delegation_root_v1(delegation: &KnowledgeDelegationV1) -> String {
|
|
807
|
+
digest_domain("delegation", &encode_cbor(&CborValue::Map(vec![
|
|
808
|
+
("payload".into(), CborValue::Bytes(delegation_payload_v1(delegation))),
|
|
809
|
+
("signature".into(), CborValue::Bytes(delegation.signature.clone())),
|
|
810
|
+
])))
|
|
811
|
+
}
|
|
812
|
+
|
|
813
|
+
pub fn authority_envelope_root_v1(envelope: &KnowledgeAuthorityEnvelopeV1) -> String {
|
|
814
|
+
digest_domain("authority-envelope", &encode_cbor(&CborValue::Map(vec![
|
|
815
|
+
("payload".into(), CborValue::Bytes(authority_envelope_payload_v1(envelope))),
|
|
816
|
+
("signature".into(), CborValue::Bytes(envelope.signature.clone())),
|
|
817
|
+
])))
|
|
818
|
+
}
|
|
819
|
+
|
|
820
|
+
pub fn authority_session_root_v1(state_root: &str, plan_root: &str, result_root: &str, authorization_root: &str, envelope_root: &str, keyring_root: Option<&str>) -> String {
|
|
821
|
+
let mut entries = vec![
|
|
822
|
+
("authorizationRoot".into(), CborValue::Text(authorization_root.into())),
|
|
823
|
+
("envelopeRoot".into(), CborValue::Text(envelope_root.into())),
|
|
824
|
+
("keyringRoot".into(), keyring_root.map_or(CborValue::Null, |value| CborValue::Text(value.into()))),
|
|
825
|
+
("planRoot".into(), CborValue::Text(plan_root.into())),
|
|
826
|
+
("resultRoot".into(), CborValue::Text(result_root.into())),
|
|
827
|
+
("stateRoot".into(), CborValue::Text(state_root.into())),
|
|
828
|
+
("version".into(), CborValue::UInt(1)),
|
|
829
|
+
];
|
|
830
|
+
digest_domain("authority-session", &encode_cbor(&CborValue::Map(std::mem::take(&mut entries))))
|
|
831
|
+
}
|
|
832
|
+
|
|
833
|
+
pub fn sync_request_payload_v1(request_id: &str, sender: &str, summary_root: &str, want_object_ids: &[String], want_event_ids: &[String], algorithm: &str, key_id: Option<&str>, keyring_root: Option<&str>, nonce: &[u8], issued_at: u64, expires_at: u64) -> Vec<u8> {
|
|
834
|
+
encode_cbor(&CborValue::Map(vec![
|
|
835
|
+
("algorithm".into(), CborValue::Text(algorithm.into())),
|
|
836
|
+
("expiresAt".into(), CborValue::UInt(expires_at)),
|
|
837
|
+
("issuedAt".into(), CborValue::UInt(issued_at)),
|
|
838
|
+
("keyId".into(), key_id.map_or(CborValue::Null, |value| CborValue::Text(value.into()))),
|
|
839
|
+
("keyringRoot".into(), keyring_root.map_or(CborValue::Null, |value| CborValue::Text(value.into()))),
|
|
840
|
+
("kind".into(), CborValue::Text("sync-request".into())),
|
|
841
|
+
("nonce".into(), CborValue::Bytes(nonce.to_vec())),
|
|
842
|
+
("requestId".into(), CborValue::Text(request_id.into())),
|
|
843
|
+
("sender".into(), CborValue::Text(sender.into())),
|
|
844
|
+
("summary".into(), CborValue::Text(summary_root.into())),
|
|
845
|
+
("version".into(), CborValue::UInt(1)),
|
|
846
|
+
("wantEventIds".into(), CborValue::Array(want_event_ids.iter().map(|value| CborValue::Text(value.clone())).collect())),
|
|
847
|
+
("wantObjectIds".into(), CborValue::Array(want_object_ids.iter().map(|value| CborValue::Text(value.clone())).collect())),
|
|
848
|
+
]))
|
|
849
|
+
}
|
|
850
|
+
|
|
851
|
+
pub fn sync_request_root_v1(request_id: &str, sender: &str, summary_root: &str, want_object_ids: &[String], want_event_ids: &[String], algorithm: &str, key_id: Option<&str>, keyring_root: Option<&str>, nonce: &[u8], issued_at: u64, expires_at: u64, signature: &[u8]) -> String {
|
|
852
|
+
digest_domain("sync-request", &encode_cbor(&CborValue::Map(vec![
|
|
853
|
+
("payload".into(), CborValue::Bytes(sync_request_payload_v1(request_id, sender, summary_root, want_object_ids, want_event_ids, algorithm, key_id, keyring_root, nonce, issued_at, expires_at))),
|
|
854
|
+
("signature".into(), CborValue::Bytes(signature.to_vec())),
|
|
855
|
+
])))
|
|
856
|
+
}
|
|
857
|
+
|
|
858
|
+
pub fn sync_response_payload_v1(request_root: &str, responder: &str, summary_root: &str, relation: &str, object_ids: &[String], event_ids: &[String], algorithm: &str, key_id: Option<&str>, keyring_root: Option<&str>, issued_at: u64, expires_at: u64) -> Vec<u8> {
|
|
859
|
+
encode_cbor(&CborValue::Map(vec![
|
|
860
|
+
("algorithm".into(), CborValue::Text(algorithm.into())),
|
|
861
|
+
("eventIds".into(), CborValue::Array(event_ids.iter().map(|value| CborValue::Text(value.clone())).collect())),
|
|
862
|
+
("expiresAt".into(), CborValue::UInt(expires_at)),
|
|
863
|
+
("issuedAt".into(), CborValue::UInt(issued_at)),
|
|
864
|
+
("keyId".into(), key_id.map_or(CborValue::Null, |value| CborValue::Text(value.into()))),
|
|
865
|
+
("keyringRoot".into(), keyring_root.map_or(CborValue::Null, |value| CborValue::Text(value.into()))),
|
|
866
|
+
("kind".into(), CborValue::Text("sync-response".into())),
|
|
867
|
+
("objectIds".into(), CborValue::Array(object_ids.iter().map(|value| CborValue::Text(value.clone())).collect())),
|
|
868
|
+
("relation".into(), CborValue::Text(relation.into())),
|
|
869
|
+
("requestRoot".into(), CborValue::Text(request_root.into())),
|
|
870
|
+
("responder".into(), CborValue::Text(responder.into())),
|
|
871
|
+
("summary".into(), CborValue::Text(summary_root.into())),
|
|
872
|
+
("version".into(), CborValue::UInt(1)),
|
|
873
|
+
]))
|
|
874
|
+
}
|
|
875
|
+
|
|
876
|
+
pub fn sync_response_root_v1(request_root: &str, responder: &str, summary_root: &str, relation: &str, object_ids: &[String], event_ids: &[String], algorithm: &str, key_id: Option<&str>, keyring_root: Option<&str>, issued_at: u64, expires_at: u64, signature: &[u8]) -> String {
|
|
877
|
+
digest_domain("sync-response", &encode_cbor(&CborValue::Map(vec![
|
|
878
|
+
("payload".into(), CborValue::Bytes(sync_response_payload_v1(request_root, responder, summary_root, relation, object_ids, event_ids, algorithm, key_id, keyring_root, issued_at, expires_at))),
|
|
879
|
+
("signature".into(), CborValue::Bytes(signature.to_vec())),
|
|
880
|
+
])))
|
|
881
|
+
}
|
|
882
|
+
|
|
883
|
+
pub fn sync_summary_root_v1(state_root: &str, commit_digest: &str, sequence: u64, parents: &[String], object_root: &str, event_root: &str, keyring_root: Option<&str>) -> String {
|
|
884
|
+
digest_domain("sync-summary", &encode_cbor(&CborValue::Map(vec![
|
|
885
|
+
("commitDigest".into(), CborValue::Text(commit_digest.into())),
|
|
886
|
+
("eventRoot".into(), CborValue::Text(event_root.into())),
|
|
887
|
+
("keyringRoot".into(), keyring_root.map_or(CborValue::Null, |value| CborValue::Text(value.into()))),
|
|
888
|
+
("objectRoot".into(), CborValue::Text(object_root.into())),
|
|
889
|
+
("parents".into(), CborValue::Array(parents.iter().map(|value| CborValue::Text(value.clone())).collect())),
|
|
890
|
+
("sequence".into(), CborValue::UInt(sequence)),
|
|
891
|
+
("stateRoot".into(), CborValue::Text(state_root.into())),
|
|
892
|
+
("version".into(), CborValue::UInt(1)),
|
|
893
|
+
])))
|
|
894
|
+
}
|
|
895
|
+
|
|
896
|
+
#[derive(Debug, Clone, PartialEq)]
|
|
897
|
+
pub struct KnowledgeAuthorityKeyV1 {
|
|
898
|
+
pub version: u64,
|
|
899
|
+
pub principal: String,
|
|
900
|
+
pub key_id: String,
|
|
901
|
+
pub algorithm: String,
|
|
902
|
+
pub public_key: Vec<u8>,
|
|
903
|
+
pub not_before: Option<u64>,
|
|
904
|
+
pub not_after: Option<u64>,
|
|
905
|
+
pub revoked_at: Option<u64>,
|
|
906
|
+
}
|
|
907
|
+
|
|
908
|
+
#[derive(Debug, Clone, PartialEq)]
|
|
909
|
+
pub struct KnowledgeKeyRotationRecordV1 {
|
|
910
|
+
pub version: u64,
|
|
911
|
+
pub kind: String,
|
|
912
|
+
pub issuer: String,
|
|
913
|
+
pub issuer_key_id: String,
|
|
914
|
+
pub principal: String,
|
|
915
|
+
pub previous_key_id: Option<String>,
|
|
916
|
+
pub key_id: String,
|
|
917
|
+
pub algorithm: String,
|
|
918
|
+
pub public_key: Vec<u8>,
|
|
919
|
+
pub not_before: u64,
|
|
920
|
+
pub not_after: Option<u64>,
|
|
921
|
+
pub revoked_at: Option<u64>,
|
|
922
|
+
pub issued_at: u64,
|
|
923
|
+
pub expires_at: u64,
|
|
924
|
+
pub signature: Vec<u8>,
|
|
925
|
+
}
|
|
926
|
+
|
|
927
|
+
#[derive(Debug, Clone, PartialEq)]
|
|
928
|
+
pub struct KnowledgeAuthorityKeyringV1 {
|
|
929
|
+
pub version: u64,
|
|
930
|
+
pub sequence: u64,
|
|
931
|
+
pub keys: Vec<KnowledgeAuthorityKeyV1>,
|
|
932
|
+
pub rotations: Vec<KnowledgeKeyRotationRecordV1>,
|
|
933
|
+
}
|
|
934
|
+
|
|
935
|
+
pub fn authority_key_payload_v1(key: &KnowledgeAuthorityKeyV1) -> Vec<u8> {
|
|
936
|
+
encode_cbor(&authority_key_value(key))
|
|
937
|
+
}
|
|
938
|
+
|
|
939
|
+
pub fn key_rotation_payload_v1(record: &KnowledgeKeyRotationRecordV1) -> Vec<u8> {
|
|
940
|
+
encode_cbor(&key_rotation_value(record, false))
|
|
941
|
+
}
|
|
942
|
+
|
|
943
|
+
pub fn key_rotation_root_v1(record: &KnowledgeKeyRotationRecordV1) -> String {
|
|
944
|
+
digest_domain("key-rotation", &encode_cbor(&CborValue::Map(vec![
|
|
945
|
+
("payload".into(), CborValue::Bytes(key_rotation_payload_v1(record))),
|
|
946
|
+
("signature".into(), CborValue::Bytes(record.signature.clone())),
|
|
947
|
+
])))
|
|
948
|
+
}
|
|
949
|
+
|
|
950
|
+
pub fn authority_keyring_root_v1(keyring: &KnowledgeAuthorityKeyringV1) -> String {
|
|
951
|
+
let mut keys = keyring.keys.iter().collect::<Vec<_>>();
|
|
952
|
+
keys.sort_by(|left, right| key_sort_key(left).cmp(&key_sort_key(right)));
|
|
953
|
+
digest_domain("authority-keyring", &encode_cbor(&CborValue::Map(vec![
|
|
954
|
+
("keys".into(), CborValue::Array(keys.iter().map(|key| CborValue::Bytes(authority_key_payload_v1(key))).collect())),
|
|
955
|
+
("rotations".into(), CborValue::Array(keyring.rotations.iter().map(|record| CborValue::Text(key_rotation_root_v1(record))).collect())),
|
|
956
|
+
("sequence".into(), CborValue::UInt(keyring.sequence)),
|
|
957
|
+
("version".into(), CborValue::UInt(keyring.version)),
|
|
958
|
+
])))
|
|
959
|
+
}
|
|
960
|
+
|
|
961
|
+
fn authority_key_value(key: &KnowledgeAuthorityKeyV1) -> CborValue {
|
|
962
|
+
let mut entries = vec![
|
|
963
|
+
("algorithm".into(), CborValue::Text(key.algorithm.clone())),
|
|
964
|
+
("keyId".into(), CborValue::Text(key.key_id.clone())),
|
|
965
|
+
("principal".into(), CborValue::Text(key.principal.clone())),
|
|
966
|
+
("publicKey".into(), CborValue::Bytes(key.public_key.clone())),
|
|
967
|
+
("version".into(), CborValue::UInt(key.version)),
|
|
968
|
+
];
|
|
969
|
+
if let Some(value) = key.not_after { entries.push(("notAfter".into(), CborValue::UInt(value))); }
|
|
970
|
+
if let Some(value) = key.not_before { entries.push(("notBefore".into(), CborValue::UInt(value))); }
|
|
971
|
+
if let Some(value) = key.revoked_at { entries.push(("revokedAt".into(), CborValue::UInt(value))); }
|
|
972
|
+
CborValue::Map(entries)
|
|
973
|
+
}
|
|
974
|
+
|
|
975
|
+
fn key_rotation_value(record: &KnowledgeKeyRotationRecordV1, include_signature: bool) -> CborValue {
|
|
976
|
+
let mut entries = vec![
|
|
977
|
+
("algorithm".into(), CborValue::Text(record.algorithm.clone())),
|
|
978
|
+
("expiresAt".into(), CborValue::UInt(record.expires_at)),
|
|
979
|
+
("issuedAt".into(), CborValue::UInt(record.issued_at)),
|
|
980
|
+
("issuer".into(), CborValue::Text(record.issuer.clone())),
|
|
981
|
+
("issuerKeyId".into(), CborValue::Text(record.issuer_key_id.clone())),
|
|
982
|
+
("keyId".into(), CborValue::Text(record.key_id.clone())),
|
|
983
|
+
("kind".into(), CborValue::Text(record.kind.clone())),
|
|
984
|
+
("notBefore".into(), CborValue::UInt(record.not_before)),
|
|
985
|
+
("publicKey".into(), CborValue::Bytes(record.public_key.clone())),
|
|
986
|
+
("principal".into(), CborValue::Text(record.principal.clone())),
|
|
987
|
+
("version".into(), CborValue::UInt(record.version)),
|
|
988
|
+
];
|
|
989
|
+
if include_signature { entries.push(("signature".into(), CborValue::Bytes(record.signature.clone()))); }
|
|
990
|
+
if let Some(value) = record.not_after { entries.push(("notAfter".into(), CborValue::UInt(value))); }
|
|
991
|
+
if let Some(value) = &record.previous_key_id { entries.push(("previousKeyId".into(), CborValue::Text(value.clone()))); }
|
|
992
|
+
if let Some(value) = record.revoked_at { entries.push(("revokedAt".into(), CborValue::UInt(value))); }
|
|
993
|
+
CborValue::Map(entries)
|
|
994
|
+
}
|
|
995
|
+
|
|
996
|
+
fn key_sort_key(key: &KnowledgeAuthorityKeyV1) -> Vec<u8> {
|
|
997
|
+
let mut out = key.principal.as_bytes().to_vec();
|
|
998
|
+
out.push(0);
|
|
999
|
+
out.extend_from_slice(key.key_id.as_bytes());
|
|
1000
|
+
out
|
|
1001
|
+
}
|
|
1002
|
+
|
|
1003
|
+
pub fn verify_knowledge_authority_envelope_v5<Resolve, Verify>(
|
|
1004
|
+
image: &KnowledgeImage,
|
|
1005
|
+
query_result: &KnowledgeQueryResultV1,
|
|
1006
|
+
policy: &KnowledgePolicyV1,
|
|
1007
|
+
authorization: &KnowledgeAuthorizationResultV1,
|
|
1008
|
+
envelope: &KnowledgeAuthorityEnvelopeV1,
|
|
1009
|
+
now: u64,
|
|
1010
|
+
resolve_key: Resolve,
|
|
1011
|
+
verify_signature: Verify,
|
|
1012
|
+
) -> Result<KnowledgeAuthorityVerificationV1, KnoloError>
|
|
1013
|
+
where
|
|
1014
|
+
Resolve: Fn(&str, &str, Option<&str>) -> Option<Vec<u8>>,
|
|
1015
|
+
Verify: Fn(&str, &[u8], &[u8], &[u8]) -> bool,
|
|
1016
|
+
{
|
|
1017
|
+
verify_knowledge_authority_envelope_with_keyring_root_v5(image, query_result, policy, authorization, envelope, now, None, resolve_key, verify_signature)
|
|
1018
|
+
}
|
|
1019
|
+
|
|
1020
|
+
pub fn verify_knowledge_authority_envelope_with_keyring_root_v5<Resolve, Verify>(
|
|
1021
|
+
image: &KnowledgeImage,
|
|
1022
|
+
query_result: &KnowledgeQueryResultV1,
|
|
1023
|
+
policy: &KnowledgePolicyV1,
|
|
1024
|
+
authorization: &KnowledgeAuthorizationResultV1,
|
|
1025
|
+
envelope: &KnowledgeAuthorityEnvelopeV1,
|
|
1026
|
+
now: u64,
|
|
1027
|
+
expected_keyring_root: Option<&str>,
|
|
1028
|
+
resolve_key: Resolve,
|
|
1029
|
+
verify_signature: Verify,
|
|
1030
|
+
) -> Result<KnowledgeAuthorityVerificationV1, KnoloError>
|
|
1031
|
+
where
|
|
1032
|
+
Resolve: Fn(&str, &str, Option<&str>) -> Option<Vec<u8>>,
|
|
1033
|
+
Verify: Fn(&str, &[u8], &[u8], &[u8]) -> bool,
|
|
1034
|
+
{
|
|
1035
|
+
let expected_authorization = evaluate_knowledge_query_policy_v5(image, query_result, policy, &authorization.principal, &authorization.action)?;
|
|
1036
|
+
if &expected_authorization != authorization { return Err(KnoloError::InvalidPack("V5 authorization result is not reproducible".into())); }
|
|
1037
|
+
if envelope.version != 1 || envelope.issuer.is_empty() || envelope.subject.is_empty() || envelope.algorithm.is_empty() || envelope.key_id.as_deref() == Some("") || envelope.issued_at > now || now >= envelope.expires_at { return Err(KnoloError::InvalidPack("invalid V5 authority envelope window".into())); }
|
|
1038
|
+
if envelope.keyring_root.is_some() && envelope.keyring_root.as_deref() != expected_keyring_root { return Err(KnoloError::InvalidPack("V5 authority keyring root mismatch".into())); }
|
|
1039
|
+
if authorization.state_root != image.state_root || envelope.authorization_root != authorization.authorization_root || envelope.subject != authorization.principal { return Err(KnoloError::InvalidPack("V5 authority binding mismatch".into())); }
|
|
1040
|
+
if envelope.delegations.len() > 8 { return Err(KnoloError::InvalidPack("V5 authority delegation depth exceeded".into())); }
|
|
1041
|
+
verify_authority_signature(&envelope.issuer, &envelope.algorithm, envelope.key_id.as_deref(), &authority_envelope_payload_v1(envelope), &envelope.signature, &resolve_key, &verify_signature)?;
|
|
1042
|
+
let mut previous = envelope.issuer.clone();
|
|
1043
|
+
for delegation in &envelope.delegations {
|
|
1044
|
+
if delegation.version != 1 || delegation.delegator.is_empty() || delegation.delegatee.is_empty() || delegation.algorithm.is_empty() || delegation.key_id.as_deref() == Some("") || delegation.action != authorization.action || delegation.issued_at > now || now >= delegation.expires_at || delegation.delegator != previous { return Err(KnoloError::InvalidPack("invalid V5 delegation chain".into())); }
|
|
1045
|
+
verify_authority_signature(&delegation.delegator, &delegation.algorithm, delegation.key_id.as_deref(), &delegation_payload_v1(delegation), &delegation.signature, &resolve_key, &verify_signature)?;
|
|
1046
|
+
previous = delegation.delegatee.clone();
|
|
1047
|
+
}
|
|
1048
|
+
if !envelope.delegations.is_empty() && previous != envelope.subject { return Err(KnoloError::InvalidPack("V5 delegation chain does not reach subject".into())); }
|
|
1049
|
+
Ok(KnowledgeAuthorityVerificationV1 { envelope_root: authority_envelope_root_v1(envelope), issuer: envelope.issuer.clone(), subject: envelope.subject.clone(), authorization_root: authorization.authorization_root.clone(), keyring_root: envelope.keyring_root.clone(), delegation_depth: envelope.delegations.len() })
|
|
1050
|
+
}
|
|
1051
|
+
|
|
1052
|
+
fn verify_authority_signature<Resolve, Verify>(principal: &str, algorithm: &str, key_id: Option<&str>, message: &[u8], signature: &[u8], resolve_key: &Resolve, verify_signature: &Verify) -> Result<(), KnoloError>
|
|
1053
|
+
where
|
|
1054
|
+
Resolve: Fn(&str, &str, Option<&str>) -> Option<Vec<u8>>,
|
|
1055
|
+
Verify: Fn(&str, &[u8], &[u8], &[u8]) -> bool,
|
|
1056
|
+
{
|
|
1057
|
+
let key = resolve_key(principal, algorithm, key_id).ok_or_else(|| KnoloError::InvalidPack("missing V5 authority key".into()))?;
|
|
1058
|
+
if !verify_signature(algorithm, &key, message, signature) { return Err(KnoloError::InvalidPack("V5 authority signature verification failed".into())); }
|
|
1059
|
+
Ok(())
|
|
1060
|
+
}
|
|
1061
|
+
|
|
1062
|
+
fn authorize_policy_hit(hit: &KnowledgeQueryHitV1, policy: &KnowledgePolicyV1, principal: &str, action: &str) -> bool {
|
|
1063
|
+
let matches = policy.rules.iter().filter(|rule| rule.action == action && rule.principal.as_deref().is_none_or(|value| value == principal) && rule.kind.as_deref().is_none_or(|value| value.to_lowercase() == hit.kind)).collect::<Vec<_>>();
|
|
1064
|
+
if matches.iter().any(|rule| rule.effect == "deny") { return false; }
|
|
1065
|
+
if matches.iter().any(|rule| rule.effect == "allow") { return true; }
|
|
1066
|
+
policy.default == "allow"
|
|
1067
|
+
}
|
|
1068
|
+
|
|
1069
|
+
fn policy_rule_value(rule: &KnowledgePolicyRuleV1) -> CborValue {
|
|
1070
|
+
let mut entries = vec![("action".into(), CborValue::Text(rule.action.clone())), ("effect".into(), CborValue::Text(rule.effect.clone()))];
|
|
1071
|
+
if let Some(kind) = &rule.kind { entries.push(("kind".into(), CborValue::Text(kind.to_lowercase()))); }
|
|
1072
|
+
if let Some(principal) = &rule.principal { entries.push(("principal".into(), CborValue::Text(principal.clone()))); }
|
|
1073
|
+
CborValue::Map(entries)
|
|
1074
|
+
}
|
|
1075
|
+
|
|
1076
|
+
fn policy_rule_key(rule: &KnowledgePolicyRuleV1) -> String { format!("{}\0{}\0{}\0{}", rule.effect, rule.action, rule.principal.as_deref().unwrap_or(""), rule.kind.as_deref().unwrap_or("").to_lowercase()) }
|
|
1077
|
+
|
|
1078
|
+
pub fn parse_knowledge_query_v5(expression: &str) -> Result<KnowledgeQueryPlanV1, KnoloError> {
|
|
1079
|
+
let tokens = lex_knowledge_query(expression)?;
|
|
1080
|
+
let mut cursor = 0usize;
|
|
1081
|
+
expect_query_word(&tokens, &mut cursor, "FROM")?;
|
|
1082
|
+
let kind_token = query_word(&tokens, &mut cursor, "FROM requires an object kind or *")?;
|
|
1083
|
+
let kind = if kind_token == "*" { None } else { Some(normalize_query_text(&kind_token)) };
|
|
1084
|
+
if kind.as_deref() == Some("") { return Err(KnoloError::InvalidPack("empty V5 EQL object kind".into())); }
|
|
1085
|
+
|
|
1086
|
+
let mut filters = Vec::new();
|
|
1087
|
+
if query_is_word(tokens.get(cursor), "WHERE") {
|
|
1088
|
+
cursor += 1;
|
|
1089
|
+
loop {
|
|
1090
|
+
let field = query_word(&tokens, &mut cursor, "WHERE requires a field")?;
|
|
1091
|
+
let field = normalize_query_field(&field)?;
|
|
1092
|
+
if !matches!(tokens.get(cursor), Some(QueryToken::Equals)) {
|
|
1093
|
+
return Err(KnoloError::InvalidPack("V5 EQL WHERE only supports =".into()));
|
|
1094
|
+
}
|
|
1095
|
+
cursor += 1;
|
|
1096
|
+
let value = parse_query_literal(tokens.get(cursor))?;
|
|
1097
|
+
cursor += 1;
|
|
1098
|
+
filters.push(KnowledgeQueryFilterV1 { field, value });
|
|
1099
|
+
if query_is_word(tokens.get(cursor), "AND") {
|
|
1100
|
+
cursor += 1;
|
|
1101
|
+
continue;
|
|
1102
|
+
}
|
|
1103
|
+
break;
|
|
1104
|
+
}
|
|
1105
|
+
}
|
|
1106
|
+
|
|
1107
|
+
let mut search = None;
|
|
1108
|
+
if query_is_word(tokens.get(cursor), "SEARCH") {
|
|
1109
|
+
cursor += 1;
|
|
1110
|
+
let value = match tokens.get(cursor) {
|
|
1111
|
+
Some(QueryToken::String(value)) => normalize_query_text(value),
|
|
1112
|
+
_ => return Err(KnoloError::InvalidPack("V5 EQL SEARCH requires a quoted string".into())),
|
|
1113
|
+
};
|
|
1114
|
+
cursor += 1;
|
|
1115
|
+
if value.is_empty() { return Err(KnoloError::InvalidPack("empty V5 EQL SEARCH".into())); }
|
|
1116
|
+
search = Some(value);
|
|
1117
|
+
}
|
|
1118
|
+
|
|
1119
|
+
let mut limit = 100usize;
|
|
1120
|
+
if query_is_word(tokens.get(cursor), "LIMIT") {
|
|
1121
|
+
cursor += 1;
|
|
1122
|
+
let value = query_word(&tokens, &mut cursor, "LIMIT requires a positive integer")?;
|
|
1123
|
+
limit = value.parse::<usize>().map_err(|_| KnoloError::InvalidPack("invalid V5 EQL LIMIT".into()))?;
|
|
1124
|
+
if !(1..=1000).contains(&limit) { return Err(KnoloError::InvalidPack("V5 EQL LIMIT must be between 1 and 1000".into())); }
|
|
1125
|
+
}
|
|
1126
|
+
if cursor != tokens.len() { return Err(KnoloError::InvalidPack("unexpected V5 EQL token".into())); }
|
|
1127
|
+
|
|
1128
|
+
filters.sort_by(|left, right| left.field.cmp(&right.field).then_with(|| query_scalar_key(&left.value).cmp(&query_scalar_key(&right.value))));
|
|
1129
|
+
Ok(KnowledgeQueryPlanV1 { kind, filters, search, limit })
|
|
1130
|
+
}
|
|
1131
|
+
|
|
1132
|
+
pub fn query_knowledge_image_v5(image: &KnowledgeImage, expression: &str) -> Result<KnowledgeQueryResultV1, KnoloError> {
|
|
1133
|
+
let plan = parse_knowledge_query_v5(expression)?;
|
|
1134
|
+
query_knowledge_plan_v5(image, &plan)
|
|
1135
|
+
}
|
|
1136
|
+
|
|
1137
|
+
fn query_knowledge_plan_v5(image: &KnowledgeImage, plan: &KnowledgeQueryPlanV1) -> Result<KnowledgeQueryResultV1, KnoloError> {
|
|
1138
|
+
let plan_root = digest_domain("query-plan", &encode_cbor(&query_plan_value(plan)));
|
|
1139
|
+
let mut objects = image.objects.iter().filter(|object| query_matches_object(object, plan)).collect::<Vec<_>>();
|
|
1140
|
+
objects.sort_by(|left, right| left.id.cmp(&right.id));
|
|
1141
|
+
let hits = objects.into_iter().take(plan.limit).map(|object| KnowledgeQueryHitV1 { object_id: object.id.clone(), kind: object.kind.clone() }).collect::<Vec<_>>();
|
|
1142
|
+
let object_ids = CborValue::Array(hits.iter().map(|hit| CborValue::Text(hit.object_id.clone())).collect());
|
|
1143
|
+
let result_value = CborValue::Map(vec![
|
|
1144
|
+
("objectIds".into(), object_ids),
|
|
1145
|
+
("planRoot".into(), CborValue::Text(plan_root.clone())),
|
|
1146
|
+
("stateRoot".into(), CborValue::Text(image.state_root.clone())),
|
|
1147
|
+
]);
|
|
1148
|
+
let result_root = digest_domain("query-result", &encode_cbor(&result_value));
|
|
1149
|
+
Ok(KnowledgeQueryResultV1 { state_root: image.state_root.clone(), plan_root, hits, result_root })
|
|
1150
|
+
}
|
|
1151
|
+
|
|
1152
|
+
fn query_plan_value(plan: &KnowledgeQueryPlanV1) -> CborValue {
|
|
1153
|
+
CborValue::Map(vec![
|
|
1154
|
+
("filters".into(), CborValue::Array(plan.filters.iter().map(|filter| CborValue::Map(vec![
|
|
1155
|
+
("field".into(), CborValue::Text(filter.field.clone())),
|
|
1156
|
+
("op".into(), CborValue::Text("=".into())),
|
|
1157
|
+
("value".into(), filter.value.clone()),
|
|
1158
|
+
])).collect())),
|
|
1159
|
+
("kind".into(), plan.kind.clone().map(CborValue::Text).unwrap_or(CborValue::Null)),
|
|
1160
|
+
("limit".into(), CborValue::UInt(plan.limit as u64)),
|
|
1161
|
+
("search".into(), plan.search.clone().map(CborValue::Text).unwrap_or(CborValue::Null)),
|
|
1162
|
+
("source".into(), CborValue::Text("knowledge-image-v5".into())),
|
|
1163
|
+
("version".into(), CborValue::UInt(1)),
|
|
1164
|
+
])
|
|
1165
|
+
}
|
|
1166
|
+
|
|
1167
|
+
fn query_matches_object(object: &KnowledgeObjectV1, plan: &KnowledgeQueryPlanV1) -> bool {
|
|
1168
|
+
if plan.kind.as_deref().is_some_and(|kind| kind != object.kind) { return false; }
|
|
1169
|
+
for filter in &plan.filters {
|
|
1170
|
+
let actual = if filter.field == "id" { Some(CborValue::Text(object.id.clone())) } else if filter.field == "kind" { Some(CborValue::Text(object.kind.clone())) } else { object.meta_map_value(&filter.field[5..]).cloned() };
|
|
1171
|
+
if !query_scalars_equal(actual.as_ref(), Some(&filter.value)) { return false; }
|
|
1172
|
+
}
|
|
1173
|
+
if let Some(search) = &plan.search {
|
|
1174
|
+
let text = normalize_query_text(&String::from_utf8_lossy(&object.bytes));
|
|
1175
|
+
if !search.split(' ').all(|term| text.contains(term)) { return false; }
|
|
1176
|
+
}
|
|
1177
|
+
true
|
|
1178
|
+
}
|
|
1179
|
+
|
|
1180
|
+
trait KnowledgeObjectMeta {
|
|
1181
|
+
fn meta_map_value(&self, key: &str) -> Option<&CborValue>;
|
|
1182
|
+
}
|
|
1183
|
+
|
|
1184
|
+
impl KnowledgeObjectMeta for KnowledgeObjectV1 {
|
|
1185
|
+
fn meta_map_value(&self, key: &str) -> Option<&CborValue> {
|
|
1186
|
+
match &self.meta { CborValue::Map(entries) => entries.iter().find(|(name, _)| name == key).map(|(_, value)| value), _ => None }
|
|
1187
|
+
}
|
|
1188
|
+
}
|
|
1189
|
+
|
|
1190
|
+
#[derive(Debug, Clone, PartialEq)]
|
|
1191
|
+
enum QueryToken { Word(String), String(String), Equals }
|
|
1192
|
+
|
|
1193
|
+
fn lex_knowledge_query(expression: &str) -> Result<Vec<QueryToken>, KnoloError> {
|
|
1194
|
+
let chars = expression.chars().collect::<Vec<_>>();
|
|
1195
|
+
let mut tokens = Vec::new();
|
|
1196
|
+
let mut cursor = 0usize;
|
|
1197
|
+
while cursor < chars.len() {
|
|
1198
|
+
while chars.get(cursor).is_some_and(|ch| ch.is_whitespace()) { cursor += 1; }
|
|
1199
|
+
if cursor >= chars.len() { break; }
|
|
1200
|
+
if chars[cursor] == '=' { tokens.push(QueryToken::Equals); cursor += 1; continue; }
|
|
1201
|
+
if chars[cursor] == '"' {
|
|
1202
|
+
cursor += 1;
|
|
1203
|
+
let mut value = String::new();
|
|
1204
|
+
let mut closed = false;
|
|
1205
|
+
while cursor < chars.len() {
|
|
1206
|
+
let ch = chars[cursor]; cursor += 1;
|
|
1207
|
+
if ch == '"' { closed = true; break; }
|
|
1208
|
+
if ch == '\\' {
|
|
1209
|
+
let escaped = *chars.get(cursor).ok_or_else(|| KnoloError::InvalidPack("unterminated V5 EQL string".into()))?; cursor += 1;
|
|
1210
|
+
if escaped != '"' && escaped != '\\' { return Err(KnoloError::InvalidPack("unsupported V5 EQL escape".into())); }
|
|
1211
|
+
value.push(escaped);
|
|
1212
|
+
} else { value.push(ch); }
|
|
1213
|
+
}
|
|
1214
|
+
if !closed { return Err(KnoloError::InvalidPack("unterminated V5 EQL string".into())); }
|
|
1215
|
+
tokens.push(QueryToken::String(value));
|
|
1216
|
+
continue;
|
|
1217
|
+
}
|
|
1218
|
+
let start = cursor;
|
|
1219
|
+
while cursor < chars.len() && !chars[cursor].is_whitespace() && chars[cursor] != '=' { cursor += 1; }
|
|
1220
|
+
let value = chars[start..cursor].iter().collect::<String>();
|
|
1221
|
+
if value.is_empty() || !value.chars().all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '_' | '.' | '*' | '-')) { return Err(KnoloError::InvalidPack("invalid V5 EQL token".into())); }
|
|
1222
|
+
tokens.push(QueryToken::Word(value));
|
|
1223
|
+
}
|
|
1224
|
+
Ok(tokens)
|
|
1225
|
+
}
|
|
1226
|
+
|
|
1227
|
+
fn expect_query_word(tokens: &[QueryToken], cursor: &mut usize, expected: &str) -> Result<(), KnoloError> {
|
|
1228
|
+
if !query_is_word(tokens.get(*cursor), expected) { return Err(KnoloError::InvalidPack(format!("V5 EQL query must start with {expected}"))); }
|
|
1229
|
+
*cursor += 1;
|
|
1230
|
+
Ok(())
|
|
1231
|
+
}
|
|
1232
|
+
|
|
1233
|
+
fn query_word(tokens: &[QueryToken], cursor: &mut usize, message: &str) -> Result<String, KnoloError> {
|
|
1234
|
+
match tokens.get(*cursor) { Some(QueryToken::Word(value)) => { *cursor += 1; Ok(value.clone()) }, _ => Err(KnoloError::InvalidPack(message.into())) }
|
|
1235
|
+
}
|
|
1236
|
+
|
|
1237
|
+
fn query_is_word(token: Option<&QueryToken>, expected: &str) -> bool { matches!(token, Some(QueryToken::Word(value)) if value.eq_ignore_ascii_case(expected)) }
|
|
1238
|
+
|
|
1239
|
+
fn normalize_query_field(field: &str) -> Result<String, KnoloError> {
|
|
1240
|
+
let lower = field.to_ascii_lowercase();
|
|
1241
|
+
if lower == "id" || lower == "kind" { return Ok(lower); }
|
|
1242
|
+
if lower.strip_prefix("meta.").is_some_and(|key| !key.is_empty() && key.chars().all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '_' | '-'))) { return Ok(lower); }
|
|
1243
|
+
Err(KnoloError::InvalidPack(format!("unsupported V5 EQL field: {field}")))
|
|
1244
|
+
}
|
|
1245
|
+
|
|
1246
|
+
fn parse_query_literal(token: Option<&QueryToken>) -> Result<CborValue, KnoloError> {
|
|
1247
|
+
match token {
|
|
1248
|
+
Some(QueryToken::String(value)) => Ok(CborValue::Text(normalize_query_text(value))),
|
|
1249
|
+
Some(QueryToken::Word(value)) if value == "true" => Ok(CborValue::Bool(true)),
|
|
1250
|
+
Some(QueryToken::Word(value)) if value == "false" => Ok(CborValue::Bool(false)),
|
|
1251
|
+
Some(QueryToken::Word(value)) if value == "null" => Ok(CborValue::Null),
|
|
1252
|
+
Some(QueryToken::Word(value)) if value.parse::<i64>().is_ok() => {
|
|
1253
|
+
let number = value.parse::<i64>().unwrap();
|
|
1254
|
+
if number >= 0 { Ok(CborValue::UInt(number as u64)) } else { Ok(CborValue::NInt(number)) }
|
|
1255
|
+
}
|
|
1256
|
+
_ => Err(KnoloError::InvalidPack("V5 EQL WHERE requires a scalar literal".into())),
|
|
1257
|
+
}
|
|
1258
|
+
}
|
|
1259
|
+
|
|
1260
|
+
fn normalize_query_text(value: &str) -> String { value.to_lowercase().split_whitespace().collect::<Vec<_>>().join(" ") }
|
|
1261
|
+
fn query_scalars_equal(actual: Option<&CborValue>, expected: Option<&CborValue>) -> bool {
|
|
1262
|
+
match (actual, expected) {
|
|
1263
|
+
(Some(CborValue::Text(left)), Some(CborValue::Text(right))) => normalize_query_text(left) == *right,
|
|
1264
|
+
_ => actual == expected,
|
|
1265
|
+
}
|
|
1266
|
+
}
|
|
1267
|
+
fn query_scalar_key(value: &CborValue) -> String {
|
|
1268
|
+
match value {
|
|
1269
|
+
CborValue::Null => "null:".into(),
|
|
1270
|
+
CborValue::Bool(value) => format!("boolean:{value}"),
|
|
1271
|
+
CborValue::UInt(value) => format!("number:{value}"),
|
|
1272
|
+
CborValue::NInt(value) => format!("number:{value}"),
|
|
1273
|
+
CborValue::Text(value) => format!("string:{value}"),
|
|
1274
|
+
other => format!("{other:?}"),
|
|
1275
|
+
}
|
|
1276
|
+
}
|
|
1277
|
+
|
|
1278
|
+
pub fn migrate_v4_to_v5(bytes: &[u8]) -> Result<MigrationResult, KnoloError> {
|
|
1279
|
+
let pack = if bytes.starts_with(b"KNLOV4\0\0") {
|
|
1280
|
+
parse_v4_pack(bytes)?
|
|
1281
|
+
} else {
|
|
1282
|
+
mount_pack_from_bytes(bytes)?
|
|
1283
|
+
};
|
|
1284
|
+
build_migration_image(&pack, &digest_from_raw(&sha256(bytes)))
|
|
1285
|
+
}
|
|
1286
|
+
|
|
1287
|
+
fn parse_v4_pack(bytes: &[u8]) -> Result<Pack, KnoloError> {
|
|
1288
|
+
if bytes.len() < 192 || &bytes[0..8] != b"KNLOV4\0\0" { return Err(KnoloError::InvalidPack("invalid V4 pack".into())); }
|
|
1289
|
+
let header_length = u32::from_le_bytes(bytes[12..16].try_into().unwrap()) as usize;
|
|
1290
|
+
let manifest_length = u32::from_le_bytes(bytes[20..24].try_into().unwrap()) as usize;
|
|
1291
|
+
let directory_length = u32::from_le_bytes(bytes[24..28].try_into().unwrap()) as usize;
|
|
1292
|
+
if header_length > bytes.len() || 192usize.checked_add(manifest_length).and_then(|n| n.checked_add(directory_length)) != Some(header_length) { return Err(KnoloError::InvalidPack("invalid V4 directory bounds".into())); }
|
|
1293
|
+
let manifest = bytes.get(192..192 + manifest_length).ok_or_else(|| KnoloError::InvalidPack("V4 manifest exceeds file".into()))?;
|
|
1294
|
+
if read_ascii_digest(&bytes[32..112]) != digest_from_raw(&sha256(manifest)) || read_ascii_digest(&bytes[112..192]) != digest_from_raw(&sha256(&bytes[header_length..])) { return Err(KnoloError::InvalidPack("V4 pack or manifest digest mismatch".into())); }
|
|
1295
|
+
let directory = std::str::from_utf8(&bytes[192 + manifest_length..header_length]).map_err(|_| KnoloError::InvalidPack("V4 directory is not UTF-8".into()))?;
|
|
1296
|
+
let (metadata_offset, metadata_length, metadata_digest) = v4_section_bounds(directory, "metadata")?;
|
|
1297
|
+
let metadata_end = metadata_offset.checked_add(metadata_length).ok_or_else(|| KnoloError::InvalidPack("V4 metadata overflow".into()))?;
|
|
1298
|
+
if metadata_end > bytes.len() || metadata_digest != digest_from_raw(&sha256(&bytes[metadata_offset..metadata_end])) { return Err(KnoloError::InvalidPack("V4 metadata digest mismatch".into())); }
|
|
1299
|
+
let metadata_json = std::str::from_utf8(&bytes[metadata_offset..metadata_end]).map_err(|_| KnoloError::InvalidPack("V4 metadata is not UTF-8".into()))?.to_string();
|
|
1300
|
+
let meta = parse_meta(&metadata_json)?;
|
|
1301
|
+
let claims_json = match v4_section_bounds(directory, "claims") {
|
|
1302
|
+
Ok((offset, length, digest)) => {
|
|
1303
|
+
let end = offset.checked_add(length).ok_or_else(|| KnoloError::InvalidPack("V4 claims overflow".into()))?;
|
|
1304
|
+
if end > bytes.len() || digest != digest_from_raw(&sha256(&bytes[offset..end])) { return Err(KnoloError::InvalidPack("V4 claims digest mismatch".into())); }
|
|
1305
|
+
Some(std::str::from_utf8(&bytes[offset..end]).map_err(|_| KnoloError::InvalidPack("V4 claims are not UTF-8".into()))?.to_string())
|
|
1306
|
+
}
|
|
1307
|
+
Err(_) => None,
|
|
1308
|
+
};
|
|
1309
|
+
let (chunks_offset, chunks_length, chunks_digest) = v4_section_bounds(directory, "chunks")?;
|
|
1310
|
+
let chunks_end = chunks_offset.checked_add(chunks_length).ok_or_else(|| KnoloError::InvalidPack("V4 chunks overflow".into()))?;
|
|
1311
|
+
if chunks_end > bytes.len() { return Err(KnoloError::InvalidPack("V4 chunks exceed file".into())); }
|
|
1312
|
+
if chunks_digest != digest_from_raw(&sha256(&bytes[chunks_offset..chunks_end])) { return Err(KnoloError::InvalidPack("V4 chunks digest mismatch".into())); }
|
|
1313
|
+
let parsed = parse_blocks(std::str::from_utf8(&bytes[chunks_offset..chunks_end]).map_err(|_| KnoloError::InvalidPack("V4 chunks are not UTF-8".into()))?)?;
|
|
1314
|
+
let count = parsed.texts.len();
|
|
1315
|
+
Ok(Pack { meta: PackMeta { stats: PackStats { docs: count, blocks: count, ..meta.stats }, ..meta }, lexicon: HashMap::new(), postings: Vec::new(), blocks: parsed.texts, headings: parsed.headings, doc_ids: parsed.doc_ids, namespaces: parsed.namespaces, block_token_lens: parsed.lens, metadata_json, claims_json })
|
|
1316
|
+
}
|
|
1317
|
+
|
|
1318
|
+
fn v4_section_bounds(directory: &str, name: &str) -> Result<(usize, usize, String), KnoloError> {
|
|
1319
|
+
let marker = format!("\"name\":\"{name}\"");
|
|
1320
|
+
let start = directory.find(&marker).ok_or_else(|| KnoloError::InvalidPack(format!("missing V4 section: {name}")))?;
|
|
1321
|
+
let entry = &directory[start..];
|
|
1322
|
+
let offset = json_number_after(entry, "\"offset\":")? as usize;
|
|
1323
|
+
let length = json_number_after(entry, "\"length\":")? as usize;
|
|
1324
|
+
let digest = json_string_after(entry, "\"digest\":\"")?;
|
|
1325
|
+
Ok((offset, length, digest))
|
|
1326
|
+
}
|
|
1327
|
+
|
|
1328
|
+
fn json_number_after(input: &str, marker: &str) -> Result<u64, KnoloError> {
|
|
1329
|
+
let start = input.find(marker).ok_or_else(|| KnoloError::InvalidPack("missing V4 directory number".into()))? + marker.len();
|
|
1330
|
+
let digits = input[start..].chars().take_while(|ch| ch.is_ascii_digit()).collect::<String>();
|
|
1331
|
+
digits.parse::<u64>().map_err(|_| KnoloError::InvalidPack("invalid V4 directory number".into()))
|
|
1332
|
+
}
|
|
1333
|
+
|
|
1334
|
+
fn json_string_after(input: &str, marker: &str) -> Result<String, KnoloError> {
|
|
1335
|
+
let start = input.find(marker).ok_or_else(|| KnoloError::InvalidPack("missing V4 directory digest".into()))? + marker.len();
|
|
1336
|
+
let end = input[start..].find('"').ok_or_else(|| KnoloError::InvalidPack("invalid V4 directory digest".into()))?;
|
|
1337
|
+
Ok(input[start..start + end].to_string())
|
|
1338
|
+
}
|
|
1339
|
+
|
|
1340
|
+
fn json_field_raw(input: &str, key: &str) -> Option<String> {
|
|
1341
|
+
let marker = format!("\"{key}\":");
|
|
1342
|
+
let start = input.find(&marker)? + marker.len();
|
|
1343
|
+
let bytes = input.as_bytes();
|
|
1344
|
+
let first = *bytes.get(start)?;
|
|
1345
|
+
if first == b'{' || first == b'[' {
|
|
1346
|
+
let open = first;
|
|
1347
|
+
let close = if open == b'{' { b'}' } else { b']' };
|
|
1348
|
+
let mut depth = 0usize;
|
|
1349
|
+
let mut in_string = false;
|
|
1350
|
+
let mut escaped = false;
|
|
1351
|
+
for index in start..bytes.len() {
|
|
1352
|
+
let byte = bytes[index];
|
|
1353
|
+
if in_string {
|
|
1354
|
+
if escaped { escaped = false; } else if byte == b'\\' { escaped = true; } else if byte == b'"' { in_string = false; }
|
|
1355
|
+
continue;
|
|
1356
|
+
}
|
|
1357
|
+
if byte == b'"' { in_string = true; continue; }
|
|
1358
|
+
if byte == open { depth += 1; }
|
|
1359
|
+
if byte == close { depth -= 1; if depth == 0 { return Some(input[start..=index].to_string()); } }
|
|
1360
|
+
}
|
|
1361
|
+
None
|
|
1362
|
+
} else {
|
|
1363
|
+
let end = input[start..].find(',').map(|offset| start + offset).unwrap_or(input.len());
|
|
1364
|
+
Some(input[start..end].trim().to_string())
|
|
1365
|
+
}
|
|
1366
|
+
}
|
|
1367
|
+
|
|
1368
|
+
fn read_ascii_digest(bytes: &[u8]) -> String {
|
|
1369
|
+
let end = bytes.iter().position(|byte| *byte == 0).unwrap_or(bytes.len());
|
|
1370
|
+
String::from_utf8_lossy(&bytes[..end]).to_string()
|
|
1371
|
+
}
|
|
1372
|
+
|
|
1373
|
+
#[derive(Clone)]
|
|
1374
|
+
struct MigrationObject { id: String, kind: String, bytes: Vec<u8>, meta: CborValue }
|
|
1375
|
+
|
|
1376
|
+
fn build_migration_image(pack: &Pack, source_digest: &str) -> Result<MigrationResult, KnoloError> {
|
|
1377
|
+
let actor = "knolo-v4-migrator";
|
|
1378
|
+
let mut objects = Vec::new();
|
|
1379
|
+
let mut mappings = Vec::new();
|
|
1380
|
+
for (index, text) in pack.blocks.iter().enumerate() {
|
|
1381
|
+
let source_bytes = text.as_bytes().to_vec();
|
|
1382
|
+
let source_meta = CborValue::Map(vec![
|
|
1383
|
+
("docId".into(), pack.doc_ids.get(index).and_then(|value| value.clone()).map(CborValue::Text).unwrap_or(CborValue::Null)),
|
|
1384
|
+
("legacyBlockId".into(), CborValue::UInt(index as u64)),
|
|
1385
|
+
("namespace".into(), pack.namespaces.get(index).and_then(|value| value.clone()).map(CborValue::Text).unwrap_or(CborValue::Null)),
|
|
1386
|
+
]);
|
|
1387
|
+
let source_body = CborValue::Map(vec![("bytes".into(), CborValue::Bytes(source_bytes.clone())), ("kind".into(), CborValue::Text("source".into())), ("meta".into(), source_meta.clone())]);
|
|
1388
|
+
let source_id = digest_domain("object", &encode_cbor(&source_body));
|
|
1389
|
+
let chunk_meta = CborValue::Map(vec![
|
|
1390
|
+
("docId".into(), pack.doc_ids.get(index).and_then(|value| value.clone()).map(CborValue::Text).unwrap_or(CborValue::Null)),
|
|
1391
|
+
("heading".into(), pack.headings.get(index).and_then(|value| value.clone()).map(CborValue::Text).unwrap_or(CborValue::Null)),
|
|
1392
|
+
("legacyBlockId".into(), CborValue::UInt(index as u64)),
|
|
1393
|
+
("namespace".into(), pack.namespaces.get(index).and_then(|value| value.clone()).map(CborValue::Text).unwrap_or(CborValue::Null)),
|
|
1394
|
+
("sourceObject".into(), CborValue::Text(source_id.clone())),
|
|
1395
|
+
("span".into(), CborValue::Map(vec![("end".into(), CborValue::UInt(text.len() as u64)), ("start".into(), CborValue::UInt(0))])),
|
|
1396
|
+
]);
|
|
1397
|
+
let chunk_body = CborValue::Map(vec![("bytes".into(), CborValue::Bytes(source_bytes.clone())), ("kind".into(), CborValue::Text("chunk".into())), ("meta".into(), chunk_meta.clone())]);
|
|
1398
|
+
let chunk_id = digest_domain("object", &encode_cbor(&chunk_body));
|
|
1399
|
+
objects.push(MigrationObject { id: source_id.clone(), kind: "source".into(), bytes: source_bytes.clone(), meta: source_meta });
|
|
1400
|
+
objects.push(MigrationObject { id: chunk_id.clone(), kind: "chunk".into(), bytes: source_bytes, meta: chunk_meta });
|
|
1401
|
+
mappings.push(MigrationMapping { legacy_block_id: index, source_object: source_id, chunk_object: chunk_id });
|
|
1402
|
+
}
|
|
1403
|
+
if let Some(claims_json) = &pack.claims_json {
|
|
1404
|
+
let bytes = claims_json.as_bytes().to_vec();
|
|
1405
|
+
let meta = CborValue::Map(vec![("encoding".into(), CborValue::Text("json-v4".into())), ("version".into(), CborValue::UInt(1))]);
|
|
1406
|
+
let body = CborValue::Map(vec![("bytes".into(), CborValue::Bytes(bytes.clone())), ("kind".into(), CborValue::Text("claims".into())), ("meta".into(), meta.clone())]);
|
|
1407
|
+
objects.push(MigrationObject { id: digest_domain("object", &encode_cbor(&body)), kind: "claims".into(), bytes, meta });
|
|
1408
|
+
}
|
|
1409
|
+
if let Some(agents_json) = json_field_raw(&pack.metadata_json, "agents") {
|
|
1410
|
+
let bytes = agents_json.as_bytes().to_vec();
|
|
1411
|
+
let meta = CborValue::Map(vec![("encoding".into(), CborValue::Text("json-v4".into())), ("version".into(), CborValue::UInt(1))]);
|
|
1412
|
+
let body = CborValue::Map(vec![("bytes".into(), CborValue::Bytes(bytes.clone())), ("kind".into(), CborValue::Text("agents".into())), ("meta".into(), meta.clone())]);
|
|
1413
|
+
objects.push(MigrationObject { id: digest_domain("object", &encode_cbor(&body)), kind: "agents".into(), bytes, meta });
|
|
1414
|
+
}
|
|
1415
|
+
let metadata_meta = CborValue::Map(vec![("sourceDigest".into(), CborValue::Text(source_digest.into())), ("sourceVersion".into(), CborValue::UInt(pack.meta.version as u64))]); let metadata_bytes = encode_cbor(&metadata_meta); let metadata_body = CborValue::Map(vec![("bytes".into(), CborValue::Bytes(metadata_bytes.clone())), ("kind".into(), CborValue::Text("metadata".into())), ("meta".into(), metadata_meta.clone())]);
|
|
1416
|
+
objects.push(MigrationObject { id: digest_domain("object", &encode_cbor(&metadata_body)), kind: "metadata".into(), bytes: metadata_bytes, meta: metadata_meta });
|
|
1417
|
+
objects.sort_by(|left, right| left.id.cmp(&right.id));
|
|
1418
|
+
let object_values = objects.iter().map(|object| CborValue::Map(vec![("bytes".into(), CborValue::Bytes(object.bytes.clone())), ("id".into(), CborValue::Text(object.id.clone())), ("kind".into(), CborValue::Text(object.kind.clone())), ("meta".into(), object.meta.clone())])).collect::<Vec<_>>();
|
|
1419
|
+
let object_payload = encode_cbor(&CborValue::Array(object_values));
|
|
1420
|
+
let object_segment_digest = digest_domain("segment", &object_payload);
|
|
1421
|
+
let object_ids = objects.iter().map(|object| CborValue::Text(object.id.clone())).collect::<Vec<_>>();
|
|
1422
|
+
let parents = CborValue::Array(Vec::new());
|
|
1423
|
+
let transaction_id = digest_domain("transaction", &encode_cbor(&CborValue::Map(vec![("actor".into(), CborValue::Text(actor.into())), ("objects".into(), CborValue::Array(object_ids.clone())), ("parents".into(), parents.clone())])));
|
|
1424
|
+
let mut events = Vec::new();
|
|
1425
|
+
for (index, object) in objects.iter().enumerate() {
|
|
1426
|
+
let event_kind = if object.kind == "chunk" { "document.put".to_string() } else { format!("{}.put", object.kind) };
|
|
1427
|
+
let event_body = CborValue::Map(vec![("actor".into(), CborValue::Text(actor.into())), ("actorCounter".into(), CborValue::UInt(index as u64 + 1)), ("kind".into(), CborValue::Text(event_kind.into())), ("parents".into(), if index == 0 { CborValue::Array(Vec::new()) } else { CborValue::Array(vec![CborValue::Text(transaction_id.clone())]) }), ("payload".into(), CborValue::Text(object.id.clone())), ("provenance".into(), CborValue::Map(vec![("objectId".into(), CborValue::Text(object.id.clone()))])), ("target".into(), CborValue::Text(object.id.clone())), ("transactionId".into(), CborValue::Text(transaction_id.clone())), ("version".into(), CborValue::UInt(1))]);
|
|
1428
|
+
let event_id = digest_domain("event", &encode_cbor(&event_body));
|
|
1429
|
+
events.push((event_id, event_body));
|
|
1430
|
+
}
|
|
1431
|
+
events.sort_by(|left, right| left.0.cmp(&right.0));
|
|
1432
|
+
let event_values = events.iter().map(|(id, body)| { let mut entries = match body { CborValue::Map(entries) => entries.clone(), _ => Vec::new() }; entries.push(("id".into(), CborValue::Text(id.clone()))); CborValue::Map(entries) }).collect::<Vec<_>>();
|
|
1433
|
+
let event_payload = encode_cbor(&CborValue::Array(event_values));
|
|
1434
|
+
let event_segment_digest = digest_domain("segment", &event_payload);
|
|
1435
|
+
let event_ids = events.iter().map(|(id, _)| CborValue::Text(id.clone())).collect::<Vec<_>>();
|
|
1436
|
+
let object_root = digest_domain("object-root", &encode_cbor(&CborValue::Array(object_ids.clone())));
|
|
1437
|
+
let event_root = digest_domain("event-root", &encode_cbor(&CborValue::Array(event_ids.clone())));
|
|
1438
|
+
let lexical_view = digest_domain("view", &encode_cbor(&CborValue::Map(vec![("kind".into(), CborValue::Text("lexical".into())), ("objectIds".into(), CborValue::Array(object_ids.clone()))])));
|
|
1439
|
+
let commit_value = CborValue::Map(vec![("actor".into(), CborValue::Text(actor.into())), ("eventRoot".into(), CborValue::Text(event_root.clone())), ("eventSegmentDigest".into(), CborValue::Text(event_segment_digest)), ("objectRoot".into(), CborValue::Text(object_root.clone())), ("objectSegmentDigest".into(), CborValue::Text(object_segment_digest)), ("parents".into(), parents), ("policyRoot".into(), CborValue::Text(digest_domain("policy", &encode_cbor(&CborValue::Map(vec![("default".into(), CborValue::Text("deny".into()))]))))), ("runtimeContract".into(), CborValue::Text(digest_domain("runtime", &encode_cbor(&CborValue::Map(vec![("codec".into(), CborValue::Text("cbor-v1".into())), ("format".into(), CborValue::UInt(5))]))))), ("schemaRoot".into(), CborValue::Text(digest_domain("schema", &encode_cbor(&CborValue::Map(vec![("version".into(), CborValue::UInt(1))]))))), ("sequence".into(), CborValue::UInt(1)), ("transactionRoot".into(), CborValue::Text(digest_domain("transaction-root", &encode_cbor(&CborValue::Map(vec![("eventIds".into(), CborValue::Array(event_ids)), ("objectIds".into(), CborValue::Array(object_ids)), ("transactionId".into(), CborValue::Text(transaction_id))]))))), ("version".into(), CborValue::UInt(1)), ("views".into(), CborValue::Map(vec![("lexical".into(), CborValue::Text(lexical_view))]))]);
|
|
1440
|
+
let commit_payload = encode_cbor(&commit_value); let commit_digest = digest_domain("commit", &commit_payload); let state = digest_domain("state", &digest_raw(&commit_digest));
|
|
1441
|
+
let object_segment = encode_v5_segment(1, &object_payload); let event_segment = encode_v5_segment(2, &event_payload); let commit_segment = encode_v5_segment(3, &commit_payload); let commit_offset = V5_DATA_START + object_segment.len() + event_segment.len();
|
|
1442
|
+
let mut image = vec![0u8; V5_DATA_START + object_segment.len() + event_segment.len() + commit_segment.len()]; image[0..8].copy_from_slice(KNOWLEDGE_IMAGE_V5_MAGIC); image[8..10].copy_from_slice(&5u16.to_le_bytes()); image[12..14].copy_from_slice(&(V5_SUPERBLOCK_SIZE as u16).to_le_bytes()); image[V5_DATA_START..V5_DATA_START + object_segment.len()].copy_from_slice(&object_segment); image[V5_DATA_START + object_segment.len()..commit_offset].copy_from_slice(&event_segment); image[commit_offset..].copy_from_slice(&commit_segment); let superblock = encode_v5_superblock(1, commit_offset, commit_segment.len(), &commit_digest, &state); image[V5_HEADER_SIZE..V5_HEADER_SIZE + V5_SUPERBLOCK_SIZE].copy_from_slice(&superblock);
|
|
1443
|
+
let receipt_body = CborValue::Map(vec![("kind".into(), CborValue::Text("v4-to-v5-migration".into())), ("objectMappings".into(), CborValue::Array(mappings.iter().map(|mapping| CborValue::Map(vec![("chunkObject".into(), CborValue::Text(mapping.chunk_object.clone())), ("legacyBlockId".into(), CborValue::UInt(mapping.legacy_block_id as u64)), ("sourceObject".into(), CborValue::Text(mapping.source_object.clone()))])).collect())), ("sourceDigest".into(), CborValue::Text(source_digest.into())), ("sourceVersion".into(), CborValue::UInt(pack.meta.version as u64)), ("stateRoot".into(), CborValue::Text(state.clone())), ("version".into(), CborValue::UInt(1))]); let receipt_digest = digest_domain("receipt", &encode_cbor(&receipt_body)); let mut receipt_entries = match receipt_body { CborValue::Map(entries) => entries, _ => Vec::new() }; receipt_entries.push(("receiptDigest".into(), CborValue::Text(receipt_digest))); let receipt = encode_cbor(&CborValue::Map(receipt_entries));
|
|
1444
|
+
Ok(MigrationResult { image, receipt, state_root: state, mappings })
|
|
1445
|
+
}
|
|
1446
|
+
|
|
1447
|
+
fn encode_v5_segment(kind: u8, payload: &[u8]) -> Vec<u8> { let mut out = vec![0u8; V5_SEGMENT_HEADER_SIZE + payload.len()]; out[0..4].copy_from_slice(SEGMENT_MAGIC); out[4] = kind; out[5] = 1; out[8..16].copy_from_slice(&(payload.len() as u64).to_le_bytes()); out[16..48].copy_from_slice(&digest_raw(&digest_domain("segment", payload))); out[V5_SEGMENT_HEADER_SIZE..].copy_from_slice(payload); out }
|
|
1448
|
+
fn encode_v5_superblock(generation: u64, commit_offset: usize, commit_length: usize, commit_digest: &str, state: &str) -> Vec<u8> { let mut out = vec![0u8; V5_SUPERBLOCK_SIZE]; out[0..8].copy_from_slice(SUPERBLOCK_MAGIC); out[8..16].copy_from_slice(&generation.to_le_bytes()); out[16..24].copy_from_slice(&(commit_offset as u64).to_le_bytes()); out[24..32].copy_from_slice(&(commit_length as u64).to_le_bytes()); out[32..64].copy_from_slice(&digest_raw(commit_digest)); out[64..96].copy_from_slice(&digest_raw(state)); let digest = digest_raw(&digest_domain("superblock", &out[0..96])); out[96..128].copy_from_slice(&digest); out }
|
|
1449
|
+
|
|
1450
|
+
fn parse_v5_image(bytes: &[u8]) -> Result<KnowledgeImage, KnoloError> {
|
|
1451
|
+
if bytes.len() < V5_DATA_START { return Err(KnoloError::InvalidPack("V5 image is truncated".into())); }
|
|
1452
|
+
if &bytes[0..8] != KNOWLEDGE_IMAGE_V5_MAGIC { return Err(KnoloError::InvalidPack("invalid V5 image magic".into())); }
|
|
1453
|
+
if le_u16(bytes, 8)? != 5 || le_u16(bytes, 12)? as usize != V5_SUPERBLOCK_SIZE { return Err(KnoloError::InvalidPack("unsupported V5 image header".into())); }
|
|
1454
|
+
|
|
1455
|
+
let a = read_v5_superblock(bytes, V5_HEADER_SIZE, 'A');
|
|
1456
|
+
let b = read_v5_superblock(bytes, V5_HEADER_SIZE + V5_SUPERBLOCK_SIZE, 'B');
|
|
1457
|
+
let mut candidates = [a, b].into_iter().flatten().collect::<Vec<_>>();
|
|
1458
|
+
if candidates.is_empty() { return Err(KnoloError::InvalidPack("no valid V5 superblock".into())); }
|
|
1459
|
+
|
|
1460
|
+
let mut segments = Vec::new();
|
|
1461
|
+
let mut offset = V5_DATA_START;
|
|
1462
|
+
let mut required_seen = [false; 3];
|
|
1463
|
+
while offset < bytes.len() {
|
|
1464
|
+
let segment = read_v5_segment(bytes, offset)?;
|
|
1465
|
+
if !matches!(segment.kind, 1..=3) && segment.kind < 128 { return Err(KnoloError::InvalidPack("unknown non-optional V5 segment".into())); }
|
|
1466
|
+
if segment.kind <= 3 && segment.schema != 1 { return Err(KnoloError::InvalidPack("unsupported required V5 segment schema".into())); }
|
|
1467
|
+
if (1..=3).contains(&segment.kind) {
|
|
1468
|
+
if required_seen[segment.kind as usize - 1] { return Err(KnoloError::InvalidPack("duplicate required V5 segment".into())); }
|
|
1469
|
+
required_seen[segment.kind as usize - 1] = true;
|
|
1470
|
+
}
|
|
1471
|
+
offset += segment.length;
|
|
1472
|
+
segments.push(segment);
|
|
1473
|
+
}
|
|
1474
|
+
if offset != bytes.len() || required_seen.iter().any(|seen| !seen) { return Err(KnoloError::InvalidPack("missing or misaligned V5 segment".into())); }
|
|
1475
|
+
|
|
1476
|
+
let commit_segment = segments.iter().find(|segment| segment.kind == 3).ok_or_else(|| KnoloError::InvalidPack("missing V5 commit segment".into()))?;
|
|
1477
|
+
let object_segment = segments.iter().find(|segment| segment.kind == 1).ok_or_else(|| KnoloError::InvalidPack("missing V5 object segment".into()))?;
|
|
1478
|
+
let event_segment = segments.iter().find(|segment| segment.kind == 2).ok_or_else(|| KnoloError::InvalidPack("missing V5 event segment".into()))?;
|
|
1479
|
+
let commit_payload = &bytes[commit_segment.offset + V5_SEGMENT_HEADER_SIZE..commit_segment.offset + commit_segment.length];
|
|
1480
|
+
let commit_value = decode_cbor_exact(commit_payload)?;
|
|
1481
|
+
if encode_cbor(&commit_value) != commit_payload { return Err(KnoloError::InvalidPack("non-canonical V5 commit CBOR".into())); }
|
|
1482
|
+
let mut commit = parse_commit(&commit_value)?;
|
|
1483
|
+
let commit_digest = digest_domain("commit", commit_payload);
|
|
1484
|
+
let state = digest_domain("state", &digest_raw(&commit_digest));
|
|
1485
|
+
candidates.sort_by(|left, right| right.generation.cmp(&left.generation));
|
|
1486
|
+
let active = candidates.into_iter().find(|candidate| candidate.commit_offset == commit_segment.offset && candidate.commit_length == commit_segment.length && candidate.commit_digest == commit_digest && candidate.state_root == state).ok_or_else(|| KnoloError::InvalidPack("no V5 superblock points to a valid commit".into()))?;
|
|
1487
|
+
commit.state_root = state.clone();
|
|
1488
|
+
commit.commit_digest = commit_digest.clone();
|
|
1489
|
+
if commit_field(&commit_value, "objectSegmentDigest")? != object_segment.digest || commit_field(&commit_value, "eventSegmentDigest")? != event_segment.digest { return Err(KnoloError::InvalidPack("V5 segment digest mismatch".into())); }
|
|
1490
|
+
|
|
1491
|
+
let objects = parse_objects(&bytes[object_segment.offset + V5_SEGMENT_HEADER_SIZE..object_segment.offset + object_segment.length])?;
|
|
1492
|
+
let events = parse_events(&bytes[event_segment.offset + V5_SEGMENT_HEADER_SIZE..event_segment.offset + event_segment.length])?;
|
|
1493
|
+
let object_ids = CborValue::Array(objects.iter().map(|object| CborValue::Text(object.id.clone())).collect());
|
|
1494
|
+
let event_ids = CborValue::Array(events.iter().map(|event| CborValue::Text(event.id.clone())).collect());
|
|
1495
|
+
if digest_domain("object-root", &encode_cbor(&object_ids)) != commit.object_root || digest_domain("event-root", &encode_cbor(&event_ids)) != commit.event_root { return Err(KnoloError::InvalidPack("V5 object/event root mismatch".into())); }
|
|
1496
|
+
|
|
1497
|
+
Ok(KnowledgeImage { state_root: state, commit_digest, commit, objects, events, segments, active_superblock: active.slot })
|
|
1498
|
+
}
|
|
1499
|
+
|
|
1500
|
+
#[derive(Debug, Clone)]
|
|
1501
|
+
struct V5Superblock { generation: u64, commit_offset: usize, commit_length: usize, commit_digest: String, state_root: String, slot: char }
|
|
1502
|
+
|
|
1503
|
+
fn read_v5_superblock(bytes: &[u8], offset: usize, slot: char) -> Option<V5Superblock> {
|
|
1504
|
+
let raw = bytes.get(offset..offset + V5_SUPERBLOCK_SIZE)?;
|
|
1505
|
+
if raw.get(0..8)? != SUPERBLOCK_MAGIC { return None; }
|
|
1506
|
+
let generation = u64::from_le_bytes(raw.get(8..16)?.try_into().ok()?);
|
|
1507
|
+
let commit_offset = usize::try_from(u64::from_le_bytes(raw.get(16..24)?.try_into().ok()?)).ok()?;
|
|
1508
|
+
let commit_length = usize::try_from(u64::from_le_bytes(raw.get(24..32)?.try_into().ok()?)).ok()?;
|
|
1509
|
+
if commit_length < V5_SEGMENT_HEADER_SIZE || commit_offset.checked_add(commit_length)? > bytes.len() { return None; }
|
|
1510
|
+
let commit_digest = digest_from_raw(raw.get(32..64)?);
|
|
1511
|
+
let state_root = digest_from_raw(raw.get(64..96)?);
|
|
1512
|
+
if digest_from_raw(raw.get(96..128)?) != digest_domain("superblock", raw.get(0..96)?) { return None; }
|
|
1513
|
+
Some(V5Superblock { generation, commit_offset, commit_length, commit_digest, state_root, slot })
|
|
1514
|
+
}
|
|
1515
|
+
|
|
1516
|
+
fn read_v5_segment(bytes: &[u8], offset: usize) -> Result<KnowledgeImageSegment, KnoloError> {
|
|
1517
|
+
let header = read_slice(bytes, &mut offset.clone(), V5_SEGMENT_HEADER_SIZE)?;
|
|
1518
|
+
if header.get(0..4) != Some(SEGMENT_MAGIC) { return Err(KnoloError::InvalidPack("invalid V5 segment magic".into())); }
|
|
1519
|
+
let kind = header[4]; let schema = header[5]; let flags = u16::from_le_bytes([header[6], header[7]]);
|
|
1520
|
+
let payload_length = usize::try_from(u64::from_le_bytes(header[8..16].try_into().unwrap())).map_err(|_| KnoloError::InvalidPack("V5 segment length overflow".into()))?;
|
|
1521
|
+
if payload_length as u64 > V5_MAX_SEGMENT { return Err(KnoloError::InvalidPack("V5 segment exceeds safety limit".into())); }
|
|
1522
|
+
let length = V5_SEGMENT_HEADER_SIZE.checked_add(payload_length).ok_or_else(|| KnoloError::InvalidPack("V5 segment length overflow".into()))?;
|
|
1523
|
+
let payload = bytes.get(offset + V5_SEGMENT_HEADER_SIZE..offset + length).ok_or_else(|| KnoloError::InvalidPack("V5 segment exceeds file".into()))?;
|
|
1524
|
+
let digest = digest_from_raw(&header[16..48]);
|
|
1525
|
+
if digest != digest_domain("segment", payload) { return Err(KnoloError::InvalidPack("V5 segment digest mismatch".into())); }
|
|
1526
|
+
Ok(KnowledgeImageSegment { kind, schema, flags, offset, length, payload_length, digest })
|
|
1527
|
+
}
|
|
1528
|
+
|
|
1529
|
+
fn parse_objects(payload: &[u8]) -> Result<Vec<KnowledgeObjectV1>, KnoloError> {
|
|
1530
|
+
let value = decode_cbor_exact(payload)?;
|
|
1531
|
+
if encode_cbor(&value) != payload { return Err(KnoloError::InvalidPack("non-canonical V5 object CBOR".into())); }
|
|
1532
|
+
let entries = match value { CborValue::Array(items) => items, _ => return Err(KnoloError::InvalidPack("invalid V5 object segment".into())) };
|
|
1533
|
+
entries.into_iter().map(|entry| {
|
|
1534
|
+
let map = as_map(&entry)?;
|
|
1535
|
+
let id = map_text(map, "id")?; let kind = map_text(map, "kind")?; let bytes = map_bytes(map, "bytes")?; let meta = map_value(map, "meta")?.clone();
|
|
1536
|
+
let body = CborValue::Map(vec![("bytes".into(), CborValue::Bytes(bytes.clone())), ("kind".into(), CborValue::Text(kind.clone())), ("meta".into(), meta.clone())]);
|
|
1537
|
+
if digest_domain("object", &encode_cbor(&body)) != id { return Err(KnoloError::InvalidPack("V5 object identity mismatch".into())); }
|
|
1538
|
+
Ok(KnowledgeObjectV1 { id, kind, bytes, meta })
|
|
1539
|
+
}).collect()
|
|
1540
|
+
}
|
|
1541
|
+
|
|
1542
|
+
fn parse_events(payload: &[u8]) -> Result<Vec<KnowledgeEventV1>, KnoloError> {
|
|
1543
|
+
let value = decode_cbor_exact(payload)?;
|
|
1544
|
+
if encode_cbor(&value) != payload { return Err(KnoloError::InvalidPack("non-canonical V5 event CBOR".into())); }
|
|
1545
|
+
let entries = match value { CborValue::Array(items) => items, _ => return Err(KnoloError::InvalidPack("invalid V5 event segment".into())) };
|
|
1546
|
+
entries.into_iter().map(|entry| {
|
|
1547
|
+
let map = as_map(&entry)?; let id = map_text(map, "id")?; let version = map_uint(map, "version")?; if version != 1 { return Err(KnoloError::InvalidPack("unsupported V5 event version".into())); }
|
|
1548
|
+
let transaction_id = map_text(map, "transactionId")?; let actor = map_text(map, "actor")?; let actor_counter = map_uint(map, "actorCounter")?; let kind = map_text(map, "kind")?; let target = map_text(map, "target")?; let payload_id = map_text(map, "payload")?;
|
|
1549
|
+
let body = CborValue::Map(vec![("actor".into(), CborValue::Text(actor.clone())), ("actorCounter".into(), CborValue::UInt(actor_counter)), ("kind".into(), CborValue::Text(kind.clone())), ("parents".into(), map_value(map, "parents")?.clone()), ("payload".into(), CborValue::Text(payload_id.clone())), ("provenance".into(), map_value(map, "provenance")?.clone()), ("target".into(), CborValue::Text(target.clone())), ("transactionId".into(), CborValue::Text(transaction_id.clone())), ("version".into(), CborValue::UInt(1))]);
|
|
1550
|
+
if digest_domain("event", &encode_cbor(&body)) != id { return Err(KnoloError::InvalidPack("V5 event identity mismatch".into())); }
|
|
1551
|
+
Ok(KnowledgeEventV1 { id, transaction_id, actor, actor_counter, kind, target, payload: payload_id })
|
|
1552
|
+
}).collect()
|
|
1553
|
+
}
|
|
1554
|
+
|
|
1555
|
+
fn parse_commit(value: &CborValue) -> Result<KnowledgeCommitV1, KnoloError> {
|
|
1556
|
+
let map = as_map(value)?; let version = map_uint(map, "version")?; if version != 1 { return Err(KnoloError::InvalidPack("unsupported V5 commit version".into())); }
|
|
1557
|
+
Ok(KnowledgeCommitV1 { state_root: String::new(), commit_digest: String::new(), parents: map_strings(map, "parents")?, object_root: map_text(map, "objectRoot")?, event_root: map_text(map, "eventRoot")?, policy_root: map_text(map, "policyRoot")?, sequence: map_uint(map, "sequence")?, actor: map_text(map, "actor")? })
|
|
1558
|
+
}
|
|
1559
|
+
|
|
1560
|
+
fn commit_field(value: &CborValue, field: &str) -> Result<String, KnoloError> { map_text(as_map(value)?, field) }
|
|
1561
|
+
|
|
1562
|
+
fn as_map(value: &CborValue) -> Result<&[(String, CborValue)], KnoloError> { match value { CborValue::Map(entries) => Ok(entries), _ => Err(KnoloError::InvalidPack("expected V5 CBOR map".into())) } }
|
|
1563
|
+
fn map_value<'a>(map: &'a [(String, CborValue)], key: &str) -> Result<&'a CborValue, KnoloError> { map.iter().find(|(name, _)| name == key).map(|(_, value)| value).ok_or_else(|| KnoloError::InvalidPack(format!("missing V5 field: {key}"))) }
|
|
1564
|
+
fn map_text(map: &[(String, CborValue)], key: &str) -> Result<String, KnoloError> { match map_value(map, key)? { CborValue::Text(value) => Ok(value.clone()), _ => Err(KnoloError::InvalidPack(format!("V5 field is not text: {key}"))) } }
|
|
1565
|
+
fn map_uint(map: &[(String, CborValue)], key: &str) -> Result<u64, KnoloError> { match map_value(map, key)? { CborValue::UInt(value) => Ok(*value), _ => Err(KnoloError::InvalidPack(format!("V5 field is not unsigned: {key}"))) } }
|
|
1566
|
+
fn map_bytes(map: &[(String, CborValue)], key: &str) -> Result<Vec<u8>, KnoloError> { match map_value(map, key)? { CborValue::Bytes(value) => Ok(value.clone()), _ => Err(KnoloError::InvalidPack(format!("V5 field is not bytes: {key}"))) } }
|
|
1567
|
+
fn map_strings(map: &[(String, CborValue)], key: &str) -> Result<Vec<String>, KnoloError> { match map_value(map, key)? { CborValue::Array(values) => values.iter().map(|value| match value { CborValue::Text(text) => Ok(text.clone()), _ => Err(KnoloError::InvalidPack("V5 array value is not text".into())) }).collect(), _ => Err(KnoloError::InvalidPack(format!("V5 field is not an array: {key}"))) } }
|
|
1568
|
+
|
|
1569
|
+
fn decode_cbor_exact(bytes: &[u8]) -> Result<CborValue, KnoloError> { let mut cursor = 0; let value = decode_cbor(bytes, &mut cursor)?; if cursor != bytes.len() { return Err(KnoloError::InvalidPack("trailing V5 CBOR bytes".into())); } Ok(value) }
|
|
1570
|
+
fn decode_cbor(bytes: &[u8], cursor: &mut usize) -> Result<CborValue, KnoloError> {
|
|
1571
|
+
let initial = *bytes.get(*cursor).ok_or_else(|| KnoloError::InvalidPack("truncated V5 CBOR".into()))?; *cursor += 1; let major = initial >> 5; let ai = initial & 31;
|
|
1572
|
+
let length = |cursor: &mut usize| read_cbor_length(bytes, cursor, ai);
|
|
1573
|
+
match major { 0 => Ok(CborValue::UInt(length(cursor)?)), 1 => { let value = length(cursor)?; if value > i64::MAX as u64 { return Err(KnoloError::InvalidPack("V5 negative integer overflow".into())); } Ok(CborValue::NInt(-1 - value as i64)) }, 2 => { let len = usize::try_from(length(cursor)?).map_err(|_| KnoloError::InvalidPack("V5 bytes length overflow".into()))?; Ok(CborValue::Bytes(read_slice(bytes, cursor, len)?.to_vec())) }, 3 => { let len = usize::try_from(length(cursor)?).map_err(|_| KnoloError::InvalidPack("V5 text length overflow".into()))?; let value = std::str::from_utf8(read_slice(bytes, cursor, len)?).map_err(|_| KnoloError::InvalidPack("V5 text is not UTF-8".into()))?; Ok(CborValue::Text(value.to_string())) }, 4 => { let len = usize::try_from(length(cursor)?).map_err(|_| KnoloError::InvalidPack("V5 array length overflow".into()))?; let mut values = Vec::with_capacity(len); for _ in 0..len { values.push(decode_cbor(bytes, cursor)?); } Ok(CborValue::Array(values)) }, 5 => { let len = usize::try_from(length(cursor)?).map_err(|_| KnoloError::InvalidPack("V5 map length overflow".into()))?; let mut values = Vec::with_capacity(len); for _ in 0..len { let key = match decode_cbor(bytes, cursor)? { CborValue::Text(value) => value, _ => return Err(KnoloError::InvalidPack("V5 map key is not text".into())) }; if values.iter().any(|(name, _): &(String, CborValue)| name == &key) { return Err(KnoloError::InvalidPack("duplicate V5 map key".into())); } values.push((key, decode_cbor(bytes, cursor)?)); } Ok(CborValue::Map(values)) }, 7 if ai == 20 => Ok(CborValue::Bool(false)), 7 if ai == 21 => Ok(CborValue::Bool(true)), 7 if ai == 22 => Ok(CborValue::Null), _ => Err(KnoloError::InvalidPack("unsupported V5 CBOR type".into())) }
|
|
1574
|
+
}
|
|
1575
|
+
fn read_cbor_length(bytes: &[u8], cursor: &mut usize, ai: u8) -> Result<u64, KnoloError> { match ai { 0..=23 => Ok(ai as u64), 24 => Ok(read_slice(bytes, cursor, 1)?[0] as u64), 25 => Ok(u16::from_be_bytes(read_slice(bytes, cursor, 2)?.try_into().unwrap()) as u64), 26 => Ok(u32::from_be_bytes(read_slice(bytes, cursor, 4)?.try_into().unwrap()) as u64), 27 => Ok(u64::from_be_bytes(read_slice(bytes, cursor, 8)?.try_into().unwrap())), _ => Err(KnoloError::InvalidPack("indefinite V5 CBOR is not allowed".into())) } }
|
|
1576
|
+
|
|
1577
|
+
fn encode_cbor(value: &CborValue) -> Vec<u8> { let mut out = Vec::new(); encode_cbor_into(value, &mut out); out }
|
|
1578
|
+
fn encode_cbor_into(value: &CborValue, out: &mut Vec<u8>) { match value { CborValue::Null => out.push(0xf6), CborValue::Bool(false) => out.push(0xf4), CborValue::Bool(true) => out.push(0xf5), CborValue::UInt(value) => encode_cbor_length(0, *value, out), CborValue::NInt(value) => encode_cbor_length(1, (-1 - *value) as u64, out), CborValue::Bytes(value) => { encode_cbor_length(2, value.len() as u64, out); out.extend_from_slice(value); }, CborValue::Text(value) => { encode_cbor_length(3, value.len() as u64, out); out.extend_from_slice(value.as_bytes()); }, CborValue::Array(values) => { encode_cbor_length(4, values.len() as u64, out); for value in values { encode_cbor_into(value, out); } }, CborValue::Map(values) => { let mut sorted = values.iter().collect::<Vec<_>>(); sorted.sort_by(|left, right| left.0.as_bytes().cmp(right.0.as_bytes())); encode_cbor_length(5, sorted.len() as u64, out); for (key, value) in sorted { encode_cbor_into(&CborValue::Text(key.clone()), out); encode_cbor_into(value, out); } } } }
|
|
1579
|
+
fn encode_cbor_length(major: u8, value: u64, out: &mut Vec<u8>) { if value < 24 { out.push((major << 5) | value as u8); } else if value <= u8::MAX as u64 { out.extend_from_slice(&[(major << 5) | 24, value as u8]); } else if value <= u16::MAX as u64 { out.push((major << 5) | 25); out.extend_from_slice(&(value as u16).to_be_bytes()); } else if value <= u32::MAX as u64 { out.push((major << 5) | 26); out.extend_from_slice(&(value as u32).to_be_bytes()); } else { out.push((major << 5) | 27); out.extend_from_slice(&value.to_be_bytes()); } }
|
|
1580
|
+
|
|
1581
|
+
fn digest_domain(domain: &str, payload: &[u8]) -> String { let mut input = format!("knolo:{domain}:v1\0").into_bytes(); input.extend_from_slice(payload); digest_from_raw(&sha256(&input)) }
|
|
1582
|
+
fn digest_from_raw(raw: &[u8]) -> String { format!("sha256-{}", raw.iter().map(|byte| format!("{byte:02x}")).collect::<String>()) }
|
|
1583
|
+
fn digest_raw(digest: &str) -> Vec<u8> { let hex = digest.strip_prefix("sha256-").unwrap_or(""); (0..hex.len()).step_by(2).filter_map(|i| u8::from_str_radix(hex.get(i..i + 2)?, 16).ok()).collect() }
|
|
1584
|
+
|
|
1585
|
+
fn le_u16(bytes: &[u8], offset: usize) -> Result<u16, KnoloError> { Ok(u16::from_le_bytes(read_slice(bytes, &mut offset.clone(), 2)?.try_into().unwrap())) }
|
|
1586
|
+
|
|
1587
|
+
// Minimal SHA-256 implementation for the dependency-free Rust kernel.
|
|
1588
|
+
fn sha256(input: &[u8]) -> [u8; 32] {
|
|
1589
|
+
const K: [u32; 64] = [
|
|
1590
|
+
0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5,0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5,
|
|
1591
|
+
0xd807aa98,0x12835b01,0x243185be,0x550c7dc3,0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174,
|
|
1592
|
+
0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc,0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da,
|
|
1593
|
+
0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7,0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967,
|
|
1594
|
+
0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13,0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85,
|
|
1595
|
+
0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3,0xd192e819,0xd6990624,0xf40e3585,0x106aa070,
|
|
1596
|
+
0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5,0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3,
|
|
1597
|
+
0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208,0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2,
|
|
1598
|
+
];
|
|
1599
|
+
let mut data = input.to_vec(); let bit_len = (data.len() as u64) * 8; data.push(0x80); while data.len() % 64 != 56 { data.push(0); } data.extend_from_slice(&bit_len.to_be_bytes());
|
|
1600
|
+
let mut h: [u32; 8] = [0x6a09e667,0xbb67ae85,0x3c6ef372,0xa54ff53a,0x510e527f,0x9b05688c,0x1f83d9ab,0x5be0cd19];
|
|
1601
|
+
for chunk in data.chunks_exact(64) { let mut w = [0u32; 64]; for i in 0..16 { w[i] = u32::from_be_bytes(chunk[i*4..i*4+4].try_into().unwrap()); } for i in 16..64 { let s0 = w[i-15].rotate_right(7) ^ w[i-15].rotate_right(18) ^ (w[i-15] >> 3); let s1 = w[i-2].rotate_right(17) ^ w[i-2].rotate_right(19) ^ (w[i-2] >> 10); w[i] = w[i-16].wrapping_add(s0).wrapping_add(w[i-7]).wrapping_add(s1); } let mut a=h[0]; let mut b=h[1]; let mut c=h[2]; let mut d=h[3]; let mut e=h[4]; let mut f=h[5]; let mut g=h[6]; let mut hh=h[7]; for i in 0..64 { let s1=e.rotate_right(6)^e.rotate_right(11)^e.rotate_right(25); let ch=(e&f)^((!e)&g); let t1=hh.wrapping_add(s1).wrapping_add(ch).wrapping_add(K[i]).wrapping_add(w[i]); let s0=a.rotate_right(2)^a.rotate_right(13)^a.rotate_right(22); let maj=(a&b)^(a&c)^(b&c); let t2=s0.wrapping_add(maj); hh=g; g=f; f=e; e=d.wrapping_add(t1); d=c; c=b; b=a; a=t1.wrapping_add(t2); } h[0]=h[0].wrapping_add(a); h[1]=h[1].wrapping_add(b); h[2]=h[2].wrapping_add(c); h[3]=h[3].wrapping_add(d); h[4]=h[4].wrapping_add(e); h[5]=h[5].wrapping_add(f); h[6]=h[6].wrapping_add(g); h[7]=h[7].wrapping_add(hh); }
|
|
1602
|
+
let mut out = [0u8; 32]; for (i, value) in h.iter().enumerate() { out[i*4..i*4+4].copy_from_slice(&value.to_be_bytes()); } out
|
|
1603
|
+
}
|