agentic-dataset-builder 0.1.0 → 0.2.0

package/agentic_dataset/export_pi_session_to_qwen35.py

@@ -1,742 +0,0 @@
-#!/usr/bin/env python3
-from __future__ import annotations
-
-import argparse
-import json
-import statistics
-from collections import Counter, defaultdict
-from datetime import datetime, timezone
-from pathlib import Path
-from typing import Any, Dict, Iterable, Iterator, List, Optional, Sequence, Tuple
-
-from .export_pi_session import ARTIFACT_KEY, compact_text, read_jsonl
-from .export_qwen35_training import (
-    append_parquet_rows,
-    ensure_parquet_runtime,
-    record_to_parquet_row,
-    validate_record_payload,
-)
-
-FULL_GLOB = '*.full.jsonl'
-RAW_GLOB = '*.raw.jsonl'
-BATCH_SIZE = 1000
-
-
-class ConversionError(RuntimeError):
-    pass
-
-
-def parse_args(argv: Sequence[str]) -> argparse.Namespace:
-    argv_list = list(argv)
-    command = 'convert'
-    if argv_list and argv_list[0] == 'stats':
-        command = 'stats'
-        argv_list = argv_list[1:]
-
-    parser = argparse.ArgumentParser(description='Convert exported Pi sessions into local Qwen3.5 training schema.')
-    parser.add_argument('--input', required=True, help='Input full-session file or directory containing exported Pi sessions, or a generated Qwen35 export dir in stats mode.')
-
-    if command == 'stats':
-        parser.add_argument('--json', action='store_true', help='Emit machine-readable JSON.')
-        args = parser.parse_args(argv_list)
-        args.command = command
-        return args
-
-    parser.add_argument('--output-root', required=True, help='Output directory root for Qwen3.5 schema export.')
-    parser.add_argument('--output-format', choices=('jsonl', 'parquet', 'both'), default='jsonl')
-    parser.add_argument('--include-raw', action='store_true', help='Also read *.raw.jsonl files when scanning directories.')
-    parser.add_argument('--limit', type=int, default=0, help='Limit number of input files processed.')
-    args = parser.parse_args(argv_list)
-    args.command = command
-    return args
-
-
-def iter_input_files(input_path: Path, include_raw: bool) -> List[Path]:
-    if input_path.is_file():
-        return [input_path]
-    if not input_path.is_dir():
-        raise ConversionError(f'Input path does not exist: {input_path}')
-    patterns = [FULL_GLOB]
-    if include_raw:
-        patterns.append(RAW_GLOB)
-    files: List[Path] = []
-    for pattern in patterns:
-        files.extend(sorted(input_path.rglob(pattern)))
-    deduped: Dict[str, Path] = {}
-    for path in files:
-        deduped[str(path.resolve())] = path.resolve()
-    return list(deduped.values())
-
-
-def build_tree(entries: List[Dict[str, Any]]) -> Tuple[Dict[str, Dict[str, Any]], Dict[Optional[str], List[str]]]:
-    by_id: Dict[str, Dict[str, Any]] = {}
-    children: Dict[Optional[str], List[str]] = defaultdict(list)
-    for entry in entries:
-        entry_id = entry.get('id')
-        if not isinstance(entry_id, str):
-            continue
-        by_id[entry_id] = entry
-        children[entry.get('parentId')].append(entry_id)
-    return by_id, children
-
-
-def leaf_ids(by_id: Dict[str, Dict[str, Any]], children: Dict[Optional[str], List[str]]) -> List[str]:
-    leaves = [entry_id for entry_id in by_id if not children.get(entry_id)]
-    return sorted(leaves)
-
-
-def path_to_leaf(leaf_id: str, by_id: Dict[str, Dict[str, Any]]) -> List[Dict[str, Any]]:
-    ordered: List[Dict[str, Any]] = []
-    current_id: Optional[str] = leaf_id
-    while current_id is not None:
-        entry = by_id.get(current_id)
-        if entry is None:
-            break
-        ordered.append(entry)
-        parent_id = entry.get('parentId')
-        current_id = parent_id if isinstance(parent_id, str) else None
-    ordered.reverse()
-    return ordered
-
-
-def convert_content_blocks(
-    content: Any,
-    lossy_reasons: set[str],
-    unsupported_reason_prefix: str,
-) -> Any:
-    if isinstance(content, str):
-        return content
-    if not isinstance(content, list):
-        lossy_reasons.add(f'{unsupported_reason_prefix}_nonstandard_content')
-        return json.dumps(content, ensure_ascii=False, sort_keys=True)
-
-    blocks: List[Dict[str, Any]] = []
-    for block in content:
-        if not isinstance(block, dict):
-            lossy_reasons.add(f'{unsupported_reason_prefix}_non_dict_block')
-            continue
-        block_type = block.get('type')
-        if block_type == 'text':
-            blocks.append({'type': 'text', 'text': block.get('text', '')})
-        elif block_type == 'image':
-            metadata: Dict[str, Any] = {}
-            if isinstance(block.get('mimeType'), str):
-                metadata['mimeType'] = block['mimeType']
-            if isinstance(block.get('data'), str):
-                metadata['data'] = block['data']
-            blocks.append(
-                {
-                    'type': 'image',
-                    'placeholder': True,
-                    'placeholder_token': '<image>',
-                    'source_kind': 'pi_session_inline_image',
-                    'metadata': metadata or None,
-                }
-            )
-        elif block_type == 'video':
-            metadata = {}
-            if isinstance(block.get('mimeType'), str):
-                metadata['mimeType'] = block['mimeType']
-            if isinstance(block.get('data'), str):
-                metadata['data'] = block['data']
-            blocks.append(
-                {
-                    'type': 'video',
-                    'placeholder': True,
-                    'placeholder_token': '<video>',
-                    'source_kind': 'pi_session_inline_video',
-                    'metadata': metadata or None,
-                }
-            )
-        else:
-            lossy_reasons.add(f'{unsupported_reason_prefix}_unsupported_block_{block_type}')
-            blocks.append({'type': 'text', 'text': json.dumps(block, ensure_ascii=False, sort_keys=True)})
-    if not blocks:
-        return ''
-    if len(blocks) == 1 and blocks[0].get('type') == 'text':
-        return blocks[0]['text']
-    return blocks
-
-
-def convert_assistant_message(message: Dict[str, Any], lossy_reasons: set[str], tools_seen: Dict[str, Dict[str, Any]]) -> Dict[str, Any]:
-    content = message.get('content')
-    text_blocks: List[Dict[str, Any]] = []
-    reasoning_chunks: List[str] = []
-    tool_calls: List[Dict[str, Any]] = []
-
-    if isinstance(content, str):
-        text_blocks = [{'type': 'text', 'text': content}]
-    elif isinstance(content, list):
-        for block in content:
-            if not isinstance(block, dict):
-                lossy_reasons.add('assistant_non_dict_block')
-                continue
-            block_type = block.get('type')
-            if block_type == 'text':
-                text_blocks.append({'type': 'text', 'text': block.get('text', '')})
-            elif block_type == 'thinking':
-                thinking = block.get('thinking')
-                if isinstance(thinking, str) and thinking:
-                    reasoning_chunks.append(thinking)
-            elif block_type == 'toolCall':
-                name = block.get('name') or 'unknown_tool'
-                arguments = block.get('arguments') if isinstance(block.get('arguments'), dict) else {}
-                tool_calls.append(
-                    {
-                        'type': 'function',
-                        'id': block.get('id'),
-                        'function': {'name': name, 'arguments': arguments},
-                    }
-                )
-                tools_seen.setdefault(name, {'name': name})
-            elif block_type in {'image', 'video'}:
-                converted = convert_content_blocks([block], lossy_reasons, 'assistant')
-                if isinstance(converted, list):
-                    text_blocks.extend(converted)
-                elif isinstance(converted, str):
-                    text_blocks.append({'type': 'text', 'text': converted})
-            else:
-                lossy_reasons.add(f'assistant_unsupported_block_{block_type}')
-                text_blocks.append({'type': 'text', 'text': json.dumps(block, ensure_ascii=False, sort_keys=True)})
-    else:
-        lossy_reasons.add('assistant_nonstandard_content')
-        text_blocks = [{'type': 'text', 'text': json.dumps(content, ensure_ascii=False, sort_keys=True)}]
-
-    assistant_content: Any
-    if not text_blocks:
-        assistant_content = ''
-    elif len(text_blocks) == 1 and text_blocks[0].get('type') == 'text':
-        assistant_content = text_blocks[0]['text']
-    else:
-        assistant_content = text_blocks
-
-    payload: Dict[str, Any] = {'role': 'assistant', 'content': assistant_content}
-    if reasoning_chunks:
-        payload['reasoning_content'] = '\n\n'.join(reasoning_chunks)
-    if tool_calls:
-        payload['tool_calls'] = tool_calls
-    return payload
-
-
-def embedded_artifact_text(node: Dict[str, Any]) -> Optional[str]:
-    embedded = node.get(f'{ARTIFACT_KEY}Embedded')
-    if not isinstance(embedded, dict):
-        return None
-    if embedded.get('encoding') == 'utf-8' and isinstance(embedded.get('text'), str):
-        return embedded['text']
-    if embedded.get('encoding') == 'base64' and isinstance(embedded.get('base64'), str):
-        return '[binary artifact embedded as base64]'
-    return None
-
-
-def format_bash_execution(message: Dict[str, Any], lossy_reasons: set[str]) -> str:
-    output = message.get('output')
-    if not isinstance(output, str):
-        output = ''
-    full_text = embedded_artifact_text(message)
-    if full_text is None and isinstance(message.get('details'), dict):
-        full_text = embedded_artifact_text(message['details'])
-    truncated = bool(message.get('truncated'))
-    if truncated and full_text is None and isinstance(message.get(ARTIFACT_KEY), str):
-        lossy_reasons.add('missing_embedded_full_output')
-    effective_output = full_text if full_text is not None else output
-    payload = {
-        'command': message.get('command'),
-        'exit_code': message.get('exitCode'),
-        'cancelled': message.get('cancelled', False),
-        'truncated': truncated,
-        'exclude_from_context': message.get('excludeFromContext', False),
-        'output': effective_output,
-    }
-    return json.dumps(payload, ensure_ascii=False, sort_keys=True)
-
-
-def synthetic_message_from_entry(entry: Dict[str, Any], label: str, text: Optional[str], lossy_reasons: set[str]) -> Optional[Dict[str, Any]]:
-    if not isinstance(text, str) or not text.strip():
-        return None
-    lossy_reasons.add(f'synthetic_{label}_message')
-    return {'role': 'assistant', 'content': f'[{label}]\n{text.strip()}'}
-
-
-def entry_has_missing_artifact(node: Any) -> bool:
-    if isinstance(node, dict):
-        for key, value in node.items():
-            if key == ARTIFACT_KEY and isinstance(value, str) and f'{ARTIFACT_KEY}Embedded' not in node:
-                return True
-            if entry_has_missing_artifact(value):
-                return True
-    elif isinstance(node, list):
-        return any(entry_has_missing_artifact(item) for item in node)
-    return False
-
-
-def convert_entry_to_messages(
-    entry: Dict[str, Any],
-    lossy_reasons: set[str],
-    tools_seen: Dict[str, Dict[str, Any]],
-) -> List[Dict[str, Any]]:
-    entry_type = entry.get('type')
-    if entry_type == 'message':
-        message = entry.get('message')
-        if not isinstance(message, dict):
-            lossy_reasons.add('message_entry_missing_payload')
-            return []
-        role = message.get('role')
-        if role == 'user':
-            return [{'role': 'user', 'content': convert_content_blocks(message.get('content'), lossy_reasons, 'user')}]
-        if role == 'assistant':
-            return [convert_assistant_message(message, lossy_reasons, tools_seen)]
-        if role == 'toolResult':
-            tool_name = message.get('toolName')
-            if isinstance(tool_name, str) and tool_name:
-                tools_seen.setdefault(tool_name, {'name': tool_name})
-            return [
-                {
-                    'role': 'tool',
-                    'content': convert_content_blocks(message.get('content'), lossy_reasons, 'tool_result'),
-                    'tool_call_id': message.get('toolCallId'),
-                    'name': tool_name,
-                }
-            ]
-        if role == 'bashExecution':
-            tools_seen.setdefault('bash', {'name': 'bash'})
-            return [{'role': 'tool', 'content': format_bash_execution(message, lossy_reasons), 'name': 'bash'}]
-        if role == 'custom':
-            custom_type = message.get('customType') or 'custom'
-            custom_content = message.get('content')
-            converted = convert_content_blocks(custom_content, lossy_reasons, 'custom')
-            lossy_reasons.add('synthetic_custom_message')
-            return [{'role': 'assistant', 'content': f'[custom:{custom_type}]\n{converted}' if isinstance(converted, str) else converted}]
-        if role == 'branchSummary':
-            return [synthetic_message_from_entry(entry, 'branch_summary', message.get('summary'), lossy_reasons)] if message.get('summary') else []
-        if role == 'compactionSummary':
-            return [synthetic_message_from_entry(entry, 'compaction_summary', message.get('summary'), lossy_reasons)] if message.get('summary') else []
-        lossy_reasons.add(f'unsupported_message_role_{role}')
-        return [{'role': 'assistant', 'content': json.dumps(message, ensure_ascii=False, sort_keys=True)}]
-
-    if entry_type == 'branch_summary':
-        return [synthetic_message_from_entry(entry, 'branch_summary', entry.get('summary'), lossy_reasons)] if entry.get('summary') else []
-    if entry_type == 'compaction':
-        return [synthetic_message_from_entry(entry, 'compaction_summary', entry.get('summary'), lossy_reasons)] if entry.get('summary') else []
-    if entry_type == 'custom_message':
-        custom_type = entry.get('customType') or 'custom'
-        converted = convert_content_blocks(entry.get('content'), lossy_reasons, 'custom_message')
-        lossy_reasons.add('synthetic_custom_message')
-        return [{'role': 'assistant', 'content': f'[custom:{custom_type}]\n{converted}' if isinstance(converted, str) else converted}]
-    return []
-
-
-def count_blocks(content: Any) -> Dict[str, int]:
-    counts = {
-        'contains_non_text_content': False,
-        'image_block_count': 0,
-        'video_block_count': 0,
-    }
-    if isinstance(content, list):
-        for block in content:
-            if not isinstance(block, dict):
-                continue
-            block_type = block.get('type')
-            if block_type == 'image':
-                counts['contains_non_text_content'] = True
-                counts['image_block_count'] += 1
-            elif block_type == 'video':
-                counts['contains_non_text_content'] = True
-                counts['video_block_count'] += 1
-    return counts
-
-
-def compute_meta_counts(messages: List[Dict[str, Any]]) -> Dict[str, Any]:
-    counts = {
-        'request_contains_non_text_content': False,
-        'request_image_block_count': 0,
-        'request_video_block_count': 0,
-        'request_tool_call_block_count': 0,
-        'request_tool_result_block_count': 0,
-        'request_thinking_block_count': 0,
-        'response_contains_non_text_content': False,
-        'response_image_block_count': 0,
-        'response_video_block_count': 0,
-        'response_tool_call_block_count': 0,
-        'response_tool_result_block_count': 0,
-        'response_thinking_block_count': 0,
-    }
-    for message in messages:
-        role = message.get('role')
-        side = 'request' if role in {'system', 'user'} else 'response'
-        block_counts = count_blocks(message.get('content'))
-        counts[f'{side}_contains_non_text_content'] = counts[f'{side}_contains_non_text_content'] or block_counts['contains_non_text_content']
-        counts[f'{side}_image_block_count'] += block_counts['image_block_count']
-        counts[f'{side}_video_block_count'] += block_counts['video_block_count']
-        if role == 'assistant':
-            counts['response_tool_call_block_count'] += len(message.get('tool_calls') or [])
-            if isinstance(message.get('reasoning_content'), str) and message['reasoning_content'].strip():
-                counts['response_thinking_block_count'] += 1
-        if role == 'tool':
-            counts['response_tool_result_block_count'] += 1
-    return counts
-
-
-def build_record_for_path(
-    header: Dict[str, Any],
-    path_entries: List[Dict[str, Any]],
-    source_path: Path,
-    leaf_id: str,
-    branch_index: int,
-    branch_count: int,
-) -> Dict[str, Any]:
-    lossy_reasons: set[str] = set()
-    tools_seen: Dict[str, Dict[str, Any]] = {}
-    messages: List[Dict[str, Any]] = []
-    models_seen: List[str] = []
-    thinking_levels: List[str] = []
-
-    for entry in path_entries:
-        if entry_has_missing_artifact(entry):
-            lossy_reasons.add('missing_embedded_artifact')
-        if entry.get('type') == 'model_change':
-            model_id = entry.get('modelId')
-            provider = entry.get('provider')
-            if isinstance(model_id, str):
-                models_seen.append(f'{provider}/{model_id}' if provider else model_id)
-            continue
-        if entry.get('type') == 'thinking_level_change':
-            level = entry.get('thinkingLevel')
-            if isinstance(level, str):
-                thinking_levels.append(level)
-            continue
-        if entry.get('type') in {'session_info', 'label', 'custom'}:
-            continue
-        for message in convert_entry_to_messages(entry, lossy_reasons, tools_seen):
-            if message:
-                messages.append(message)
-
-    if not any(message.get('role') == 'user' for message in messages):
-        raise ConversionError(f'No user messages found on branch {leaf_id} from {source_path}')
-
-    if branch_count > 1:
-        lossy_reasons.add('session_tree_branch_selected')
-
-    export_info = header.get('exportInfo') if isinstance(header.get('exportInfo'), dict) else None
-    if isinstance(export_info, dict) and int(export_info.get('missingArtifactCount') or 0) > 0:
-        lossy_reasons.add('source_export_missing_artifacts')
-
-    if len(set(models_seen)) > 1:
-        lossy_reasons.add('multiple_models_on_branch')
-    if len(set(thinking_levels)) > 1:
-        lossy_reasons.add('multiple_thinking_levels_on_branch')
-
-    tools = list(tools_seen.values())
-    counts = compute_meta_counts(messages)
-    meta = {
-        'endpoint': 'pi/session_branch',
-        'status': 200,
-        'ts': path_entries[-1].get('timestamp') or header.get('timestamp') or '',
-        'key': header.get('id'),
-        'source': f'{source_path}#leaf={leaf_id}',
-        'requested_model': models_seen[0] if models_seen else None,
-        'actual_model': models_seen[-1] if models_seen else None,
-        'stream': False,
-        'thinking_level': thinking_levels[-1] if thinking_levels else None,
-        'reasoning_summary_mode': 'pi_session_branch',
-        'thinking_type': 'pi_session',
-        'thinking_budget_tokens': None,
-        'max_output_tokens': None,
-        'tool_spec_count': len(tools),
-        'tool_choice': {'mode': 'session_trace'},
-        'request_truncated': False,
-        'response_truncated': 'missing_embedded_full_output' in lossy_reasons,
-        'lossy_source': bool(lossy_reasons),
-        'lossy_reasons': sorted(lossy_reasons),
-        **counts,
-    }
-
-    record = {
-        'id': f"{header.get('id')}:{leaf_id}",
-        'request_id': header.get('id'),
-        'messages': messages,
-        'tools': tools,
-        'meta': meta,
-    }
-    validate_record_payload(record)
-    return record
-
-
-def convert_file(path: Path) -> Tuple[List[Dict[str, Any]], Counter]:
-    entries = read_jsonl(path)
-    header = entries[0]
-    body = entries[1:]
-    by_id, children = build_tree(body)
-    leaves = leaf_ids(by_id, children)
-    stats = Counter()
-    stats['input_files'] += 1
-    stats['branches_total'] += len(leaves)
-    records: List[Dict[str, Any]] = []
-    for index, leaf_id in enumerate(leaves, start=1):
-        branch_path = path_to_leaf(leaf_id, by_id)
-        record = build_record_for_path(header, branch_path, path, leaf_id, index, len(leaves))
-        records.append(record)
-        if record['meta']['lossy_source']:
-            stats['lossy_records'] += 1
-        else:
-            stats['strict_records'] += 1
-    return records, stats
-
-
-def load_qwen_records(input_path: Path) -> List[Dict[str, Any]]:
-    files: List[Path] = []
-    if input_path.is_file():
-        files = [input_path]
-    elif input_path.is_dir():
-        for name in ('qwen35-train.jsonl', 'qwen35-train-lossy.jsonl'):
-            candidate = input_path / name
-            if candidate.exists():
-                files.append(candidate)
-    if not files:
-        raise ConversionError(f'No Qwen35 jsonl files found under {input_path}')
-
-    records: List[Dict[str, Any]] = []
-    for path in files:
-        bucket = 'lossy' if 'lossy' in path.name else 'strict'
-        with path.open('r', encoding='utf-8') as handle:
-            for line in handle:
-                line = line.strip()
-                if not line:
-                    continue
-                record = json.loads(line)
-                record['_bucket'] = bucket
-                records.append(record)
-    return records
-
-
-def content_char_count(content: Any) -> int:
-    if isinstance(content, str):
-        return len(content)
-    if isinstance(content, list):
-        total = 0
-        for block in content:
-            if isinstance(block, dict):
-                if isinstance(block.get('text'), str):
-                    total += len(block['text'])
-                else:
-                    total += len(json.dumps(block, ensure_ascii=False, sort_keys=True))
-            else:
-                total += len(str(block))
-        return total
-    return len(json.dumps(content, ensure_ascii=False, sort_keys=True))
-
-
-def stat_summary(values: List[int]) -> Optional[Dict[str, Any]]:
-    if not values:
-        return None
-    ordered = sorted(values)
-    return {
-        'min': ordered[0],
-        'median': statistics.median(ordered),
-        'mean': round(statistics.mean(ordered), 2),
-        'max': ordered[-1],
-    }
-
-
-def build_stats_report(records: List[Dict[str, Any]], input_path: Path) -> Dict[str, Any]:
-    message_counts: List[int] = []
-    user_counts: List[int] = []
-    assistant_counts: List[int] = []
-    tool_counts: List[int] = []
-    round_counts: List[int] = []
-    tool_call_counts: List[int] = []
-    reasoning_message_counts: List[int] = []
-    reasoning_char_counts: List[int] = []
-    total_char_counts: List[int] = []
-    per_record: List[Dict[str, Any]] = []
-    assistant_total = 0
-    assistant_with_reasoning_total = 0
-
-    for record in records:
-        messages = record.get('messages', [])
-        users = [message for message in messages if message.get('role') == 'user']
-        assistants = [message for message in messages if message.get('role') == 'assistant']
-        tools = [message for message in messages if message.get('role') == 'tool']
-        reasoning_messages = [
-            message
-            for message in assistants
-            if isinstance(message.get('reasoning_content'), str) and message['reasoning_content'].strip()
-        ]
-        reasoning_chars = sum(len(message['reasoning_content']) for message in reasoning_messages)
-        tool_calls = sum(len(message.get('tool_calls') or []) for message in assistants)
-        total_chars = sum(content_char_count(message.get('content')) for message in messages)
-
-        message_counts.append(len(messages))
-        user_counts.append(len(users))
-        assistant_counts.append(len(assistants))
-        tool_counts.append(len(tools))
-        round_counts.append(len(users))
-        tool_call_counts.append(tool_calls)
-        reasoning_message_counts.append(len(reasoning_messages))
-        reasoning_char_counts.append(reasoning_chars)
-        total_char_counts.append(total_chars)
-        assistant_total += len(assistants)
-        assistant_with_reasoning_total += len(reasoning_messages)
-
-        per_record.append(
-            {
-                'id': record.get('id'),
-                'bucket': record.get('_bucket'),
-                'messages': len(messages),
-                'users': len(users),
-                'assistants': len(assistants),
-                'tools': len(tools),
-                'dialogue_rounds_est': len(users),
-                'tool_calls': tool_calls,
-                'reasoning_messages': len(reasoning_messages),
-                'reasoning_chars': reasoning_chars,
-                'content_chars': total_chars,
-                'lossy_reasons': record.get('meta', {}).get('lossy_reasons', []),
-            }
-        )
-
-    records_with_reasoning = sum(1 for count in reasoning_message_counts if count > 0)
-    report = {
-        'input': str(input_path),
-        'records': len(records),
-        'strict_records': sum(1 for record in records if record.get('_bucket') == 'strict'),
-        'lossy_records': sum(1 for record in records if record.get('_bucket') == 'lossy'),
-        'message_count': stat_summary(message_counts),
-        'user_messages': stat_summary(user_counts),
-        'assistant_messages': stat_summary(assistant_counts),
-        'tool_messages': stat_summary(tool_counts),
-        'dialogue_rounds_est': stat_summary(round_counts),
-        'assistant_tool_calls': stat_summary(tool_call_counts),
-        'assistant_reasoning_messages': stat_summary(reasoning_message_counts),
-        'reasoning_chars_total_per_record': stat_summary(reasoning_char_counts),
-        'content_chars_total': stat_summary(total_char_counts),
-        'records_with_reasoning': records_with_reasoning,
-        'records_with_reasoning_ratio': round(records_with_reasoning / len(records), 4) if records else 0.0,
-        'assistant_messages_with_reasoning': assistant_with_reasoning_total,
-        'assistant_messages_total': assistant_total,
-        'assistant_reasoning_coverage': round(assistant_with_reasoning_total / assistant_total, 4) if assistant_total else 0.0,
-        'per_record': per_record,
-    }
-    return report
-
-
-def print_stats_report(report: Dict[str, Any], as_json: bool) -> int:
-    if as_json:
-        print(json.dumps(report, ensure_ascii=False, indent=2))
-        return 0
-    print(f"input: {report['input']}")
-    print(f"records: {report['records']} (strict={report['strict_records']}, lossy={report['lossy_records']})")
-    print(f"records with reasoning: {report['records_with_reasoning']} ({report['records_with_reasoning_ratio']:.2%})")
-    print(
-        f"assistant reasoning coverage: {report['assistant_messages_with_reasoning']}/"
-        f"{report['assistant_messages_total']} ({report['assistant_reasoning_coverage']:.2%})"
-    )
-    print(f"message count: {report['message_count']}")
-    print(f"dialogue rounds est: {report['dialogue_rounds_est']}")
-    print(f"assistant tool calls: {report['assistant_tool_calls']}")
-    print(f"assistant reasoning messages: {report['assistant_reasoning_messages']}")
-    print(f"reasoning chars per record: {report['reasoning_chars_total_per_record']}")
-    print('per record:')
-    for item in report['per_record']:
-        print(
-            f"  - {item['id']} [{item['bucket']}] msgs={item['messages']} rounds={item['dialogue_rounds_est']} "
-            f"tool_calls={item['tool_calls']} reasoning_msgs={item['reasoning_messages']} reasoning_chars={item['reasoning_chars']}"
-        )
-    return 0
-
-
-def main(argv: Sequence[str] | None = None) -> int:
-    args = parse_args(argv or [])
-
-    if args.command == 'stats':
-        input_path = Path(args.input).expanduser().resolve()
-        records = load_qwen_records(input_path)
-        report = build_stats_report(records, input_path)
-        return print_stats_report(report, args.json)
-
-    ensure_parquet_runtime(args.output_format)
-
-    input_path = Path(args.input).expanduser().resolve()
-    input_files = iter_input_files(input_path, args.include_raw)
-    if args.limit > 0:
-        input_files = input_files[: args.limit]
-    if not input_files:
-        raise SystemExit('No exported Pi session files found.')
-
-    out_dir = Path(args.output_root).expanduser().resolve() / f'qwen35-pi-session-{datetime.now(timezone.utc).strftime("%Y%m%d-%H%M%S")}'
-    out_dir.mkdir(parents=True, exist_ok=True)
-    strict_path = out_dir / 'qwen35-train.jsonl'
-    lossy_path = out_dir / 'qwen35-train-lossy.jsonl'
-    invalid_path = out_dir / 'invalid-records.jsonl'
-    strict_parquet_path = out_dir / 'qwen35-train.parquet'
-    lossy_parquet_path = out_dir / 'qwen35-train-lossy.parquet'
-    manifest_path = out_dir / 'manifest.json'
-
-    stats = Counter()
-    strict_out = strict_path.open('w', encoding='utf-8') if args.output_format in {'jsonl', 'both'} else None
-    lossy_out = lossy_path.open('w', encoding='utf-8') if args.output_format in {'jsonl', 'both'} else None
-    invalid_out = invalid_path.open('w', encoding='utf-8')
-    strict_writer = None
-    lossy_writer = None
-    strict_batch: List[Dict[str, Any]] = []
-    lossy_batch: List[Dict[str, Any]] = []
-
-    try:
-        for path in input_files:
-            try:
-                records, file_stats = convert_file(path)
-                stats.update(file_stats)
-            except Exception as exc:
-                stats['invalid_files'] += 1
-                invalid_out.write(json.dumps({'path': str(path), 'error': str(exc)}, ensure_ascii=False) + '\n')
-                continue
-
-            for record in records:
-                bucket = 'lossy' if record['meta']['lossy_source'] else 'strict'
-                if bucket == 'strict':
-                    if strict_out is not None:
-                        strict_out.write(json.dumps(record, ensure_ascii=False, separators=(',', ':')) + '\n')
-                    if args.output_format in {'parquet', 'both'}:
-                        strict_batch.append(record_to_parquet_row(record))
-                        if len(strict_batch) >= BATCH_SIZE:
-                            strict_writer = append_parquet_rows(strict_writer, strict_batch, strict_parquet_path)
-                            strict_batch = []
-                else:
-                    if lossy_out is not None:
-                        lossy_out.write(json.dumps(record, ensure_ascii=False, separators=(',', ':')) + '\n')
-                    if args.output_format in {'parquet', 'both'}:
-                        lossy_batch.append(record_to_parquet_row(record))
-                        if len(lossy_batch) >= BATCH_SIZE:
-                            lossy_writer = append_parquet_rows(lossy_writer, lossy_batch, lossy_parquet_path)
-                            lossy_batch = []
-            print(json.dumps({'processed_files': stats['input_files'], **dict(stats)}, ensure_ascii=False), flush=True)
-
-        if args.output_format in {'parquet', 'both'}:
-            strict_writer = append_parquet_rows(strict_writer, strict_batch, strict_parquet_path)
-            lossy_writer = append_parquet_rows(lossy_writer, lossy_batch, lossy_parquet_path)
-    finally:
-        if strict_out is not None:
-            strict_out.close()
-        if lossy_out is not None:
-            lossy_out.close()
-        invalid_out.close()
-        if strict_writer is not None:
-            strict_writer.close()
-        if lossy_writer is not None:
-            lossy_writer.close()
-
-    manifest = {
-        'input': str(input_path),
-        'output_dir': str(out_dir),
-        'input_files': [str(path) for path in input_files],
-        'stats': dict(stats),
-        'strict_records': stats.get('strict_records', 0),
-        'lossy_records': stats.get('lossy_records', 0),
-    }
-    manifest_path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
-    print(json.dumps(manifest, ensure_ascii=False), flush=True)
-    return 0
-
-
-if __name__ == '__main__':
-    raise SystemExit(main(__import__('sys').argv[1:]))