@method-ai-hq/sdk 0.0.0-stage → 0.14.3

package/dist/packages/sdk/examples/daily-briefing/briefing_manifest.py

@@ -0,0 +1,23 @@
+"""Describe only the generated website and its explicit supporting outputs."""
+import mimetypes
+from pathlib import PurePosixPath
+from briefing_artifacts import digest, local, put
+
+
+def website_manifest(destination, inventory, title, extras):
+    destination = local(destination)
+    files = []
+    for relative, expected in inventory.items():
+        path = PurePosixPath(relative)
+        if path.is_absolute() or '..' in path.parts:
+            raise ValueError('Invalid website output path')
+        files.append({'path': 'website/' + relative, 'sha256': expected,
+                      'media_type': mimetypes.guess_type(relative)[0] or 'application/octet-stream'})
+    for name in extras:
+        if PurePosixPath(name).name != name:
+            raise ValueError('Invalid supporting output path')
+        files.append({'path': name, 'sha256': digest(destination / name),
+                      'media_type': mimetypes.guess_type(name)[0] or 'application/octet-stream'})
+    return put(destination / 'website-result.json', {
+        'schema': 'method-website/1', 'title': title, 'entrypoint': 'website/index.html', 'files': files,
+    })

package/dist/packages/sdk/examples/daily-briefing/briefing_markdown.py

@@ -0,0 +1,67 @@
+"""Read ordinary Markdown and collect source links without prescribing prose."""
+import datetime as dt
+import json
+import os
+from pathlib import Path
+import re
+import subprocess
+from zoneinfo import ZoneInfo
+
+
+def parse(text, day, marked=None, supporting=()):
+    date = dt.date.fromisoformat(day['day'])
+    ZoneInfo(day['timezone'])
+    supplements = list(supporting)
+    module = Path(marked or os.environ.get('DAILY_BRIEFING_MARKED') or Path(__file__).parent/'vendor/marked.mjs').resolve()
+    code = r'''
+import {marked} from MARKED;
+let chunks=[];for await(const c of process.stdin)chunks.push(c);
+const text=Buffer.concat(chunks).toString();
+const escape=s=>s.replaceAll('&','&amp;').replaceAll('<','&lt;').replaceAll('>','&gt;').replaceAll('"','&quot;');
+const citations=[],photos=[],local_files=[];
+const target=(href,image=false)=>{
+ if (/^(https?:|#)/.test(href)) return;
+ if (/^[a-z][a-z0-9+.-]*:/i.test(href) || href.startsWith('/')) throw Error(`Unsupported file target: ${href}`);
+ const path=decodeURIComponent(href.split(/[?#]/)[0]);
+ if (path.split(/[\\/]/).some(x=>x==='..'||x==='.git'||x.startsWith('.env'))) throw Error(`Unsafe file target: ${href}`);
+ if(path)local_files.push({path,image});
+};
+marked.use({renderer:{
+ html(token){return token.text.startsWith('<!--')?'':escape(token.text)},
+ link(token){
+  const label=this.parser.parseInline(token.tokens), href=token.href;
+  if(href.startsWith('source:')){const index=citations.length;citations.push({target:href.slice(7),label});return `<!--briefing-citation-${index}-->`}
+  target(href); return `<a href="${escape(href)}" rel="noopener noreferrer">${label}</a>`;
+ },
+ image(token){
+  if(token.href.startsWith('source:')){const index=photos.length;photos.push({id:token.href.slice(7),caption:token.text});return `<!--briefing-photo-${index}-->`}
+  target(token.href,true); return `<img src="${escape(token.href)}" alt="${escape(token.text||'')}" loading="lazy">`;
+ }
+}});
+const tokens=marked.lexer(text),blocks=[];
+for(const token of tokens){
+ if(token.type==='space')continue;
+ const one=[token];one.links=tokens.links;
+ const session=token.type==='paragraph' && token.tokens?.length===1 && token.tokens[0].type==='link' && /^sessions\.html#.+$/.test(token.tokens[0].href) ? token.tokens[0].href.split('#')[1] : undefined;
+ blocks.push({type:token.type,depth:token.depth,text:token.text||'',html:marked.parser(one),...(session?{session}: {})});
+}
+process.stdout.write(JSON.stringify({blocks,citations,photos,local_files}));
+'''.replace('MARKED', json.dumps(module.as_uri()))
+    document = json.loads(subprocess.run(['node', '--input-type=module', '-e', code], input=text, text=True, capture_output=True, check=True).stdout)
+    ranges = {}
+    for citation in document['citations']:
+        ids, sep, selection = citation.pop('target').partition('#L')
+        citation['citations'] = ids.split(',')
+        if not all(re.fullmatch(r'[A-Za-z0-9_-]+', cid) for cid in citation['citations']):
+            raise ValueError('Invalid source record ID')
+        if sep:
+            match = re.fullmatch(r'(\d+)-L(\d+)', selection)
+            if not match or len(citation['citations']) != 1:
+                raise ValueError('A passage needs one source and #Lfirst-Llast')
+            bounds = [int(match[1]), int(match[2])]
+            key = 'lines-' + '-'.join(map(str, bounds))
+            ranges.setdefault(ids, {})[key] = bounds
+            citation['source_range'] = key
+    heading = next((b['text'] for b in document['blocks'] if b['type']=='heading' and b.get('depth')==1), date.strftime('%A, %B %d').replace(' 0',' '))
+    return {**document, 'day': date.isoformat(), 'timezone': day['timezone'], 'heading': heading,
+            'title': heading, 'source_ranges': ranges, 'supplements': supplements}

package/dist/packages/sdk/examples/daily-briefing/briefing_progress.py

@@ -0,0 +1,18 @@
+"""Report public work milestones without changing the final JSON result."""
+import json
+import os
+
+
+def progress(message, completed=None, total=None, unit=None):
+    fd = os.environ.get('METHOD_PROGRESS_FD')
+    if fd is None:
+        return
+    update = {'message': message}
+    if completed is not None and total is not None:
+        update.update(completed=completed, total=total)
+    if unit:
+        update['unit'] = unit
+    try:
+        os.write(int(fd), (json.dumps(update) + '\n').encode())
+    except (OSError, ValueError):
+        pass

package/dist/packages/sdk/examples/daily-briefing/briefing_render.py

@@ -0,0 +1,71 @@
+"""Render, check, and save a briefing once. Inputs remain read-only."""
+import contextlib
+import hashlib
+import json
+from pathlib import Path
+import shutil
+import sys
+import tempfile
+
+from briefing_artifacts import digest, local, put, source_digest, tree
+from briefing_files import folder
+from briefing_manifest import website_manifest
+from briefing_validation import supporting_files, check_website
+from briefing_markdown import parse
+from briefing_website import build
+from briefing_progress import progress
+
+
+def render_and_save(args):
+    day, draft = args['selected_day'], args['draft']
+    supporting, files = supporting_files(args)
+    proposal = parse(draft, day, supporting=supporting)
+    progress('Rendering the briefing and its sources.')
+    temporary = Path(tempfile.mkdtemp(prefix='briefing-', dir=local('.')))
+    try:
+        (temporary/'briefing.md').write_text(draft)
+        for name, path in files.items(): shutil.copyfile(path, temporary/name)
+        website = temporary/'website'
+        with contextlib.redirect_stdout(sys.stderr):
+            build(folder('selected_day'), temporary/'briefing.md', website,
+                  Path(__file__).parent/'vendor/marked.mjs', day, proposal)
+        for target in proposal['local_files']:
+            relative = target['path']
+            destination = (website/relative).resolve()
+            if not destination.is_relative_to(website.resolve()):
+                raise ValueError('File target leaves the website: ' + relative)
+            if destination.is_file(): continue
+            source = files.get(relative)
+            if source is None:
+                source = (folder('selected_day')/relative).resolve()
+                if not source.is_relative_to(folder('selected_day').resolve()):
+                    raise ValueError('File target leaves the prepared folder: ' + relative)
+            if not source.is_file(): raise ValueError('Missing linked file: ' + relative)
+            destination.parent.mkdir(parents=True, exist_ok=True)
+            shutil.copyfile(source, destination)
+        check_website(website)
+        if source_digest() != day['source_digest']:
+            raise ValueError('Prepared sources changed during the run')
+        inventory = tree(website)
+        extras = ['briefing.md', *files]
+        signature = hashlib.sha256(json.dumps({'website': inventory,
+            'files': {name:digest(temporary/name) for name in extras}}, sort_keys=True).encode()).hexdigest()
+        destination = local('saved/'+signature)
+        destination.parent.mkdir(parents=True, exist_ok=True)
+        if destination.exists():
+            if tree(destination/'website') != inventory or any(digest(destination/name) != digest(temporary/name) for name in extras):
+                raise ValueError('Existing saved output changed')
+        else:
+            temporary.rename(destination)
+        published = website_manifest(destination, inventory, 'Briefing for '+day['day'], extras)
+        receipt = put(destination/'briefing.json', {'day':day['day'], 'timezone':day['timezone'],
+            'website':str((destination/'website/index.html').relative_to(local('.'))),
+            'source_digest':day['source_digest'], 'supporting_files':list(files)})
+        progress('Briefing saved. Website links are valid.')
+        return {'saved_briefing':receipt, 'published_website':published}
+    finally:
+        if temporary.exists(): shutil.rmtree(temporary)
+
+
+if __name__ == '__main__':
+    print(json.dumps(render_and_save(json.load(sys.stdin)), ensure_ascii=False))

package/dist/packages/sdk/examples/daily-briefing/briefing_sessions.py

@@ -0,0 +1,37 @@
+"""Group saved social visits for the website. This is not a time estimate."""
+import datetime as dt
+import hashlib
+import json
+
+
+def stamp(value):return dt.datetime.fromisoformat(value)
+
+
+DEFAULT_DISPLAY_GAP_SECONDS = 300
+
+def social_sessions(prepared, gap_seconds=DEFAULT_DISPLAY_GAP_SECONDS):
+    if gap_seconds <= 0: raise ValueError('The display gap must be positive')
+    if not (prepared/'sessions.json').exists(): return []
+    entries={r['entry_id']:r for r in map(json.loads,(prepared/'timeline.jsonl').read_text().splitlines())}
+    groups=[];source_sessions=json.loads((prepared/'sessions.json').read_text())
+    for session in source_sessions:
+        for social in session['social']:
+            segments=[]
+            for eid in social['event_ids']:
+                row=entries[eid]
+                if not segments or (stamp(row['time'])-stamp(segments[-1][-1]['time'])).total_seconds()>gap_seconds:segments.append([])
+                segments[-1].append(row)
+            for segment in segments:
+                sid='social-'+hashlib.sha256((session['session_id']+'|'+social['platform']+'|'+segment[0]['entry_id']).encode()).hexdigest()[:12]
+                start,end=segment[0]['time'],segment[-1]['time']
+                full=[entries[eid] for eid in session['event_ids'] if stamp(start)<=stamp(entries[eid]['time'])<=stamp(end)]
+                groups.append({'id':sid,'parent':session['session_id'],'platform':social['platform'],'stream':session['stream'],'start':start,'end':end,'social':segment,'full':full})
+    # Keep isolated uncertain Takeout records as separate rows in the overlapping view.
+    attached=set()
+    for g in groups:
+        if not g['stream'].startswith('takeout:') or len(g['social'])!=1:continue
+        candidates=[other for other in groups if other['stream'].startswith('browser:') and other['platform']==g['platform'] and stamp(other['start'])<=stamp(g['start'])<=stamp(other['end'])]
+        if len(candidates)==1:
+            candidates[0]['full']+=g['full'];attached.add(g['id'])
+    groups=[g for g in groups if g['id'] not in attached]
+    return sorted(groups,key=lambda g:(stamp(g["start"]),g["id"]))

package/dist/packages/sdk/examples/daily-briefing/briefing_times.py

@@ -0,0 +1,158 @@
+"""Calculate durations from explicit source selections. No model calls or prose writing."""
+import datetime as dt
+import itertools
+import json
+import math
+import sys
+from zoneinfo import ZoneInfo
+
+from briefing_files import folder
+
+
+def number(value):
+    if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value) or value < 0:
+        raise ValueError('Expected a finite, nonnegative number')
+    return value
+
+
+def stamp(value):
+    value = dt.datetime.fromisoformat(value)
+    if value.utcoffset() is None:
+        raise ValueError('An estimate needs a timestamp with a saved clock offset')
+    return value.astimezone(dt.timezone.utc)
+
+
+def union(ranges):
+    merged = []
+    for start, end in sorted(ranges):
+        if end <= start:
+            continue
+        if merged and start <= merged[-1][1]:
+            merged[-1] = (merged[-1][0], max(end, merged[-1][1]))
+        else:
+            merged.append((start, end))
+    return merged
+
+
+def minutes(ranges):
+    return sum((end-start).total_seconds()/60 for start, end in union(ranges))
+
+
+def clusters(points, gap):
+    groups = []
+    for point in sorted(set(points)):
+        if not groups or (point-groups[-1][-1]).total_seconds() > gap*60:
+            groups.append([])
+        groups[-1].append(point)
+    return [(group[0], group[-1]) for group in groups]
+
+
+def calculate(plan, prepared, day, timezone):
+    zone = ZoneInfo(timezone)
+    date = dt.date.fromisoformat(day)
+    lower = dt.datetime.combine(date, dt.time(), zone).astimezone(dt.timezone.utc)
+    upper = dt.datetime.combine(date+dt.timedelta(days=1), dt.time(), zone).astimezone(dt.timezone.utc)
+    records = {}
+
+    def record(cid):
+        if not isinstance(cid, str) or not cid or any(c not in 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_' for c in cid):
+            raise ValueError('Invalid record ID')
+        path = (prepared/'records'/f'{cid}.json').resolve(strict=True)
+        if not path.is_relative_to(prepared.resolve()):
+            raise ValueError('Record is outside prepared folder')
+        if cid not in records:
+            records[cid] = json.loads(path.read_text())['prepared']
+        return records[cid]
+
+    def value(ref):
+        obj = record(ref['record_id'])
+        for key in ref['field'].split('.'):
+            obj = obj[key]
+        return obj
+
+    def clipped(ranges):
+        result = []
+        for start, end in ranges:
+            if end < start:
+                raise ValueError('An interval ends before it starts')
+            start, end = max(start, lower), min(end, upper)
+            if end >= start:
+                result.append((start, end))
+        return union(result)
+
+    results, all_ranges, ids = [], {}, set()
+    for activity in plan['activities']:
+        aid = activity['id']
+        if aid in ids:
+            raise ValueError('Repeated activity ID')
+        ids.add(aid)
+        mode = activity['mode']
+        sources, ranges, extra = set(), [], {}
+        # Reasons and exclusions are agent decisions. The script retains them unchanged.
+        for exclusion in activity.get('excluded', []):
+            for cid in exclusion['record_ids']:
+                record(cid)
+        if mode == 'points':
+            gap = number(activity['gap_minutes'])
+            points = []
+            for ref in activity['points']:
+                point = stamp(value(ref))
+                if not lower <= point < upper:
+                    raise ValueError('Point is outside the selected day')
+                sources.add(ref['record_id']); points.append(point)
+            ranges = clusters(points, gap)
+            extra['gap_sensitivity_minutes'] = {str(g): minutes(clusters(points, number(g))) for g in activity.get('compare_gaps', [])}
+        elif mode == 'intervals':
+            for interval in activity['intervals']:
+                ranges.append((stamp(value(interval['start'])), stamp(value(interval['end']))))
+                sources.update([interval['start']['record_id'], interval['end']['record_id']])
+        elif mode == 'durations':
+            divisor = {'milliseconds': 60000, 'seconds': 60, 'minutes': 1}[activity['unit']]
+            refs = activity['durations']
+            keys = [(r['record_id'], r['field']) for r in refs]
+            if len(set(keys)) != len(keys):
+                raise ValueError('Repeated duration source')
+            total = sum(number(value(ref)) for ref in refs)/divisor
+            sources.update(ref['record_id'] for ref in refs)
+        else:
+            raise ValueError('Unknown calculation mode')
+        ranges = clipped(ranges)
+        if mode != 'durations':
+            total = minutes(ranges)
+            all_ranges[aid] = ranges
+        results.append({'id': aid, 'label': activity['label'], 'minutes': total,
+                        'ranges': [{'start': a.astimezone(zone).isoformat(), 'end': b.astimezone(zone).isoformat()} for a,b in ranges],
+                        'source_record_ids': sorted(sources), 'reason': activity['reason'],
+                        'excluded': activity.get('excluded', []), **extra})
+    overlaps = [{'activities': [a,b], 'minutes': minutes([(max(x,u),min(y,v)) for x,y in all_ranges[a] for u,v in all_ranges[b] if max(x,u)<min(y,v)])}
+                for a,b in itertools.combinations(all_ranges, 2)]
+    return {'day': day, 'timezone': timezone, 'activities': results, 'overlaps': overlaps,
+            'not_estimated': plan.get('not_estimated', [])}
+
+
+def explanation(result):
+    """Display the saved arithmetic and the writer's reasons without changing them."""
+    lines=['# Time estimates', '', 'Session spans and recorded durations can overlap. They do not measure continuous attention.', '']
+    for activity in result['activities']:
+        lines += ['## '+activity['label'], '', f"{activity['minutes']:.1f} minutes. "+activity['reason'], '']
+        sensitivity=activity.get('gap_sensitivity_minutes', {})
+        if sensitivity:
+            lines += ['Changing the session gap: '+', '.join(f'{gap} minutes → {value:.1f} minutes' for gap,value in sensitivity.items())+'.', '']
+        for excluded in activity.get('excluded', []):
+            lines += [excluded.get('reason', str(excluded)) if isinstance(excluded, dict) else str(excluded), '']
+    if result.get('not_estimated'):
+        lines += ['## Not estimated', '', *[(str(item.get('label') or item.get('id') or '')+': '+str(item.get('reason') or '')) if isinstance(item, dict) else str(item) for item in result['not_estimated']]]
+    return '\n'.join(lines)
+
+
+def calculate_activity_times(args):
+    from briefing_artifacts import local, put
+    plan = json.loads(args['plan'])
+    result = calculate(plan, folder('selected_day'), args['day'], args['timezone'])
+    # Calculate first so an invalid request leaves the last successful files intact.
+    files = [put(local('time-plan.json'), plan), put(local('time-estimates.json'), result)]
+    return {'estimates': json.dumps(result, ensure_ascii=False), 'files': files}
+
+
+if __name__ == '__main__':
+    print(json.dumps(calculate_activity_times(json.load(sys.stdin)), ensure_ascii=False))

package/dist/packages/sdk/examples/daily-briefing/briefing_validation.py

@@ -0,0 +1,101 @@
+"""Check draft references and website links."""
+import json
+import sys
+from html.parser import HTMLParser
+from urllib.parse import unquote, urlsplit
+
+from briefing_artifacts import digest, local
+from briefing_times import explanation
+from briefing_files import folder
+from briefing_markdown import parse
+
+
+def check_draft(draft, day, supporting=()):
+    doc=parse(draft, day, supporting=supporting)
+    records={p.stem for p in (folder('selected_day')/'records').glob('*.json')}
+    supplements={s['id'] for s in doc['supplements']}
+    if len(supplements)!=len(doc['supplements']):raise ValueError('Repeated supplement ID')
+    if records & supplements:raise ValueError('A supplement uses a source record ID')
+    allowed=records|supplements
+    for citation in doc['citations']:
+        if set(citation['citations'])-allowed:raise ValueError('Unknown citation')
+    for photo in doc['photos']:
+        if photo['id'] not in records:raise ValueError('Unknown photo record')
+    for cid,ranges in doc.get('source_ranges',{}).items():
+        text = (json.loads((folder('selected_day')/'records'/f'{cid}.json').read_text())['prepared']['text']
+                if cid in records else next(s['text'] for s in doc['supplements'] if s['id']==cid))
+        for first,last in ranges.values():
+            if not 1<=first<=last<=len(text.splitlines()):raise ValueError('Passage lines are outside the saved text')
+    return doc
+
+
+class Links(HTMLParser):
+    def __init__(self):super().__init__();self.ids=set();self.links=[];self.citations=[];self.entries=set()
+    def handle_starttag(self,tag,attrs):
+        a=dict(attrs)
+        if 'id' in a:self.ids.add(a['id'])
+        if 'data-entry' in a:self.entries.add(a['data-entry'])
+        for name in ('href','src'):
+            if name in a:self.links.append(a[name])
+        if 'data-ids' in a:self.citations.extend(json.loads(a['data-ids']))
+
+
+def check_website(root):
+    pages={}
+    for path in root.glob('*.html'):
+        page=Links();page.feed(path.read_text());pages[path.resolve()]=page
+    for path,page in pages.items():
+        for link in page.links:
+            url=urlsplit(link)
+            if url.scheme or url.netloc:continue
+            target=(path.parent/unquote(url.path)).resolve() if url.path else path
+            if not target.is_relative_to(root.resolve()) or not target.is_file():raise ValueError('Broken local website link: '+link)
+            if url.fragment and target in pages and unquote(url.fragment) not in pages[target].ids:raise ValueError('Missing passage anchor: '+link)
+        if any(not (root/'data'/f'{cid}.json').is_file() for cid in page.citations):raise ValueError('A citation has no source view')
+    groups=json.loads((root/'display-groups.json').read_text())
+    if any(g['id'] not in pages[(root/'sessions.html').resolve()].ids for g in groups):raise ValueError('A browsing group is absent from the sessions page')
+    sessions = folder('selected_day')/'sessions.json'
+    expected={eid for session in (json.loads(sessions.read_text()) if sessions.exists() else []) for social in session['social'] for eid in social['event_ids']}
+    shown={eid for group in groups for eid in group['full']}
+    if expected-shown:raise ValueError('A saved social visit is absent from the website')
+    entries=[row for name in ['timeline.jsonl','untimed.jsonl'] for row in map(json.loads,(folder('selected_day')/name).read_text().splitlines())]
+    browser_ids={row['entry_id'] for row in entries if json.loads((folder('selected_day')/'records'/f"{row['source_record_ids'][0]}.json").read_text())['prepared']['source'] in ['browser','google_takeout']}
+    if browser_ids and browser_ids-pages[(root/'browser.html').resolve()].entries:raise ValueError('A browser visit is absent from All browser visits')
+    return len(groups)
+
+
+def supporting_files(args):
+    day = args['selected_day']
+    supporting, files = [], {}
+    for ref in args.get('supporting_files', []):
+        path = local(ref['path'])
+        if digest(path) != ref['sha256']:
+            raise ValueError('A supporting file changed: ' + path.name)
+        if path.suffix not in ('.json', '.md', '.txt') or path.name in ('briefing.md', 'briefing.json', 'website-result.json') or path.name in files:
+            raise ValueError('Invalid supporting file: ' + path.name)
+        text = path.read_text()
+        if path.name == 'time-estimates.json':
+            estimates = json.loads(text)
+            if (estimates['day'], estimates['timezone']) != (day['day'], day['timezone']):
+                raise ValueError('Calculated times do not match the selected date and timezone')
+            text = explanation(estimates)
+        elif path.suffix == '.json':
+            text = '```json\n'+text+'\n```'
+        files[path.name] = path
+        supporting.append({'id': path.stem, 'title': path.stem.replace('-', ' ').capitalize(), 'text': text})
+    return supporting, files
+
+
+def check_written_briefing(args):
+    try:
+        values = {**args['inputs'], **args['outputs']}
+        supporting, files = supporting_files(values)
+        doc = check_draft(values['draft'], values['selected_day'], supporting)
+        return {'status': 'pass', 'reason': 'Source links and supporting files are valid.',
+                'evidence': [f"{len(doc['citations'])} citations checked.", f"{len(files)} supporting files checked."]}
+    except (ValueError, KeyError, OSError) as error:
+        return {'status': 'fail', 'reason': str(error), 'evidence': []}
+
+
+if __name__ == '__main__':
+    print(json.dumps(check_written_briefing(json.load(sys.stdin))))