aigroup-workflow 2.2.1 → 2.2.2

package/skills/python/pandas-pro/references/merging-joining.md

@@ -1,596 +1,596 @@
-# Merging and Joining
-
----
-
-## Overview
-
-Combining DataFrames is essential for working with relational data. This reference covers merge, join, concat, and advanced combination strategies with pandas 2.0+.
-
----
-
-## Merge (SQL-Style Joins)
-
-### Basic Merge
-
-```python
-import pandas as pd
-import numpy as np
-
-# Sample DataFrames
-employees = pd.DataFrame({
-    'emp_id': [1, 2, 3, 4, 5],
-    'name': ['Alice', 'Bob', 'Charlie', 'Diana', 'Eve'],
-    'dept_id': [101, 102, 101, 103, 102],
-})
-
-departments = pd.DataFrame({
-    'dept_id': [101, 102, 104],
-    'dept_name': ['Engineering', 'Sales', 'Marketing'],
-})
-
-# Inner join (default) - only matching rows
-result = pd.merge(employees, departments, on='dept_id')
-
-# Explicit how parameter
-result = pd.merge(employees, departments, on='dept_id', how='inner')
-```
-
-### Join Types
-
-```python
-# Inner join - only matching rows from both
-inner = pd.merge(employees, departments, on='dept_id', how='inner')
-# Result: 4 rows (emp_id 4 has dept_id 103 which doesn't exist in departments)
-
-# Left join - all rows from left, matching from right
-left = pd.merge(employees, departments, on='dept_id', how='left')
-# Result: 5 rows (Diana has NaN for dept_name)
-
-# Right join - all rows from right, matching from left
-right = pd.merge(employees, departments, on='dept_id', how='right')
-# Result: 4 rows (Marketing has no employees, but is included)
-
-# Outer join - all rows from both
-outer = pd.merge(employees, departments, on='dept_id', how='outer')
-# Result: 6 rows (includes unmatched from both sides)
-
-# Cross join - cartesian product
-cross = pd.merge(employees, departments, how='cross')
-# Result: 15 rows (5 employees x 3 departments)
-```
-
-### Merging on Different Column Names
-
-```python
-employees = pd.DataFrame({
-    'emp_id': [1, 2, 3],
-    'name': ['Alice', 'Bob', 'Charlie'],
-    'department': [101, 102, 101],
-})
-
-departments = pd.DataFrame({
-    'id': [101, 102],
-    'dept_name': ['Engineering', 'Sales'],
-})
-
-# Different column names
-result = pd.merge(
-    employees,
-    departments,
-    left_on='department',
-    right_on='id'
-)
-
-# Drop duplicate column after merge
-result = result.drop('id', axis=1)
-```
-
-### Merging on Multiple Columns
-
-```python
-sales = pd.DataFrame({
-    'region': ['East', 'East', 'West', 'West'],
-    'product': ['A', 'B', 'A', 'B'],
-    'sales': [100, 150, 120, 180],
-})
-
-targets = pd.DataFrame({
-    'region': ['East', 'East', 'West'],
-    'product': ['A', 'B', 'A'],
-    'target': [90, 140, 110],
-})
-
-# Merge on multiple columns
-result = pd.merge(sales, targets, on=['region', 'product'], how='left')
-```
-
-### Merging on Index
-
-```python
-# Set index before merge
-employees_idx = employees.set_index('emp_id')
-salaries = pd.DataFrame({
-    'emp_id': [1, 2, 3, 4],
-    'salary': [80000, 75000, 70000, 65000],
-}).set_index('emp_id')
-
-# Merge on index
-result = pd.merge(employees_idx, salaries, left_index=True, right_index=True)
-
-# Mix of column and index
-result = pd.merge(
-    employees,
-    salaries,
-    left_on='emp_id',
-    right_index=True
-)
-```
-
----
-
-## Handling Duplicate Columns
-
-### Suffixes
-
-```python
-df1 = pd.DataFrame({
-    'id': [1, 2, 3],
-    'value': [10, 20, 30],
-    'date': ['2024-01-01', '2024-01-02', '2024-01-03'],
-})
-
-df2 = pd.DataFrame({
-    'id': [1, 2, 3],
-    'value': [100, 200, 300],
-    'date': ['2024-02-01', '2024-02-02', '2024-02-03'],
-})
-
-# Default suffixes
-result = pd.merge(df1, df2, on='id')
-# Columns: id, value_x, date_x, value_y, date_y
-
-# Custom suffixes
-result = pd.merge(df1, df2, on='id', suffixes=('_jan', '_feb'))
-# Columns: id, value_jan, date_jan, value_feb, date_feb
-```
-
-### Validate Merge Cardinality
-
-```python
-# Validate merge relationships (pandas 2.0+)
-# Raises MergeError if validation fails
-
-# One-to-one: each key appears at most once in both DataFrames
-result = pd.merge(df1, df2, on='id', validate='one_to_one')  # or '1:1'
-
-# One-to-many: keys unique in left only
-result = pd.merge(employees, salaries, on='emp_id', validate='one_to_many')  # or '1:m'
-
-# Many-to-one: keys unique in right only
-result = pd.merge(salaries, employees, on='emp_id', validate='many_to_one')  # or 'm:1'
-
-# Many-to-many: no uniqueness requirement (default)
-result = pd.merge(df1, df2, on='id', validate='many_to_many')  # or 'm:m'
-```
-
-### Indicator Column
-
-```python
-# Add indicator column showing source of each row
-result = pd.merge(
-    employees,
-    departments,
-    on='dept_id',
-    how='outer',
-    indicator=True
-)
-# _merge column values: 'left_only', 'right_only', 'both'
-
-# Custom indicator name
-result = pd.merge(
-    employees,
-    departments,
-    on='dept_id',
-    how='outer',
-    indicator='source'
-)
-
-# Filter by indicator
-left_only = result[result['_merge'] == 'left_only']
-both = result[result['_merge'] == 'both']
-```
-
----
-
-## Join (Index-Based)
-
-### DataFrame.join()
-
-```python
-# join() is for index-based joining (simpler syntax)
-employees = pd.DataFrame({
-    'name': ['Alice', 'Bob', 'Charlie'],
-    'dept_id': [101, 102, 101],
-}, index=[1, 2, 3])
-
-salaries = pd.DataFrame({
-    'salary': [80000, 75000, 70000],
-    'bonus': [5000, 4000, 3500],
-}, index=[1, 2, 3])
-
-# Join on index
-result = employees.join(salaries)
-
-# Join types (same as merge)
-result = employees.join(salaries, how='left')
-result = employees.join(salaries, how='outer')
-```
-
-### Join on Column to Index
-
-```python
-employees = pd.DataFrame({
-    'name': ['Alice', 'Bob', 'Charlie'],
-    'dept_id': [101, 102, 101],
-})
-
-departments = pd.DataFrame({
-    'dept_name': ['Engineering', 'Sales'],
-}, index=[101, 102])
-
-# Join left column to right index
-result = employees.join(departments, on='dept_id')
-```
-
-### Join Multiple DataFrames
-
-```python
-df1 = pd.DataFrame({'a': [1, 2]}, index=['x', 'y'])
-df2 = pd.DataFrame({'b': [3, 4]}, index=['x', 'y'])
-df3 = pd.DataFrame({'c': [5, 6]}, index=['x', 'y'])
-
-# Join multiple at once
-result = df1.join([df2, df3])
-
-# With suffixes for duplicate columns
-result = df1.join([df2, df3], lsuffix='_1', rsuffix='_2')
-```
-
----
-
-## Concat (Stacking DataFrames)
-
-### Vertical Concatenation (Row-wise)
-
-```python
-# Stack DataFrames vertically
-df1 = pd.DataFrame({
-    'name': ['Alice', 'Bob'],
-    'age': [25, 30],
-})
-
-df2 = pd.DataFrame({
-    'name': ['Charlie', 'Diana'],
-    'age': [35, 28],
-})
-
-# Basic concat (axis=0 is default)
-result = pd.concat([df1, df2])
-
-# Reset index
-result = pd.concat([df1, df2], ignore_index=True)
-
-# Keep track of source
-result = pd.concat([df1, df2], keys=['source1', 'source2'])
-# Creates MultiIndex
-```
-
-### Horizontal Concatenation (Column-wise)
-
-```python
-names = pd.DataFrame({'name': ['Alice', 'Bob', 'Charlie']})
-ages = pd.DataFrame({'age': [25, 30, 35]})
-salaries = pd.DataFrame({'salary': [50000, 60000, 70000]})
-
-# Concat columns (axis=1)
-result = pd.concat([names, ages, salaries], axis=1)
-```
-
-### Handling Mismatched Columns
-
-```python
-df1 = pd.DataFrame({
-    'name': ['Alice', 'Bob'],
-    'age': [25, 30],
-})
-
-df2 = pd.DataFrame({
-    'name': ['Charlie', 'Diana'],
-    'salary': [70000, 65000],
-})
-
-# Outer join (default) - include all columns
-result = pd.concat([df1, df2])
-# age and salary columns have NaN where not present
-
-# Inner join - only common columns
-result = pd.concat([df1, df2], join='inner')
-# Only 'name' column
-```
-
-### Concat with Verification
-
-```python
-# Verify no index overlap
-try:
-    result = pd.concat([df1, df2], verify_integrity=True)
-except ValueError as e:
-    print(f"Index overlap detected: {e}")
-
-# Alternative: use ignore_index
-result = pd.concat([df1, df2], ignore_index=True)
-```
-
----
-
-## Combine and Update
-
-### combine_first() - Fill Gaps
-
-```python
-# Fill NaN values from another DataFrame
-df1 = pd.DataFrame({
-    'A': [1, np.nan, 3],
-    'B': [np.nan, 2, 3],
-}, index=['a', 'b', 'c'])
-
-df2 = pd.DataFrame({
-    'A': [10, 20, 30],
-    'B': [10, 20, 30],
-}, index=['a', 'b', 'c'])
-
-# Fill NaN in df1 with values from df2
-result = df1.combine_first(df2)
-# A: [1, 20, 3], B: [10, 2, 3]
-```
-
-### update() - In-Place Update
-
-```python
-df1 = pd.DataFrame({
-    'A': [1, 2, 3],
-    'B': [4, 5, 6],
-}, index=['a', 'b', 'c'])
-
-df2 = pd.DataFrame({
-    'A': [10, 20],
-    'B': [40, 50],
-}, index=['a', 'b'])
-
-# Update df1 with values from df2 (in-place)
-df1.update(df2)
-# df1 now has A: [10, 20, 3], B: [40, 50, 6]
-
-# Only update where df2 has non-NaN
-df1.update(df2, overwrite=False)  # Don't overwrite existing values
-```
-
----
-
-## Advanced Merge Patterns
-
-### Merge with Aggregation
-
-```python
-# Merge and aggregate in one operation
-orders = pd.DataFrame({
-    'order_id': [1, 2, 3, 4],
-    'customer_id': [101, 102, 101, 103],
-    'amount': [100, 200, 150, 300],
-})
-
-customers = pd.DataFrame({
-    'customer_id': [101, 102, 103],
-    'name': ['Alice', 'Bob', 'Charlie'],
-})
-
-# Get customer summary
-customer_summary = orders.groupby('customer_id').agg(
-    total_orders=('order_id', 'count'),
-    total_amount=('amount', 'sum'),
-).reset_index()
-
-# Merge with customer info
-result = pd.merge(customers, customer_summary, on='customer_id')
-```
-
-### Merge Asof (Nearest Match)
-
-```python
-# Merge on nearest key (useful for time series)
-trades = pd.DataFrame({
-    'time': pd.to_datetime(['2024-01-01 10:00:01', '2024-01-01 10:00:03', '2024-01-01 10:00:05']),
-    'ticker': ['AAPL', 'AAPL', 'AAPL'],
-    'price': [150.0, 151.0, 150.5],
-})
-
-quotes = pd.DataFrame({
-    'time': pd.to_datetime(['2024-01-01 10:00:00', '2024-01-01 10:00:02', '2024-01-01 10:00:04']),
-    'ticker': ['AAPL', 'AAPL', 'AAPL'],
-    'bid': [149.5, 150.5, 150.0],
-    'ask': [150.5, 151.5, 151.0],
-})
-
-# Merge asof - find nearest quote for each trade
-result = pd.merge_asof(
-    trades.sort_values('time'),
-    quotes.sort_values('time'),
-    on='time',
-    by='ticker',
-    direction='backward'  # Use most recent quote
-)
-```
-
-### Conditional Merge
-
-```python
-# Merge with conditions beyond key equality
-# First merge, then filter
-
-products = pd.DataFrame({
-    'product_id': [1, 2, 3],
-    'name': ['Widget', 'Gadget', 'Gizmo'],
-    'category': ['A', 'B', 'A'],
-})
-
-discounts = pd.DataFrame({
-    'category': ['A', 'A', 'B'],
-    'min_qty': [10, 50, 20],
-    'discount': [0.05, 0.10, 0.08],
-})
-
-# Cross merge then filter
-merged = pd.merge(products, discounts, on='category')
-# Then apply quantity-based filtering as needed
-```
-
----
-
-## Performance Considerations
-
-### Pre-sorting for Merge
-
-```python
-# Sort keys before merge for better performance
-df1 = df1.sort_values('key')
-df2 = df2.sort_values('key')
-
-# Merge sorted DataFrames
-result = pd.merge(df1, df2, on='key')
-```
-
-### Index Alignment
-
-```python
-# Using index for merge is often faster than columns
-df1 = df1.set_index('key')
-df2 = df2.set_index('key')
-
-# Join on index
-result = df1.join(df2)
-```
-
-### Memory-Efficient Merge
-
-```python
-# For large DataFrames, reduce memory before merge
-# Convert to appropriate types
-df1['key'] = df1['key'].astype('int32')  # Instead of int64
-df1['category'] = df1['category'].astype('category')
-
-# Select only needed columns
-cols_needed = ['key', 'value1', 'value2']
-result = pd.merge(df1[cols_needed], df2[cols_needed], on='key')
-```
-
----
-
-## Common Merge Patterns
-
-### Left Join with Null Check
-
-```python
-# Find unmatched rows after left join
-result = pd.merge(employees, departments, on='dept_id', how='left')
-unmatched = result[result['dept_name'].isna()]
-```
-
-### Anti-Join (Rows Not in Other)
-
-```python
-# Find employees NOT in a specific department list
-dept_list = [101, 102]
-
-# Method 1: Using isin
-not_in_depts = employees[~employees['dept_id'].isin(dept_list)]
-
-# Method 2: Using merge with indicator
-merged = pd.merge(
-    employees,
-    pd.DataFrame({'dept_id': dept_list}),
-    on='dept_id',
-    how='left',
-    indicator=True
-)
-not_in_depts = merged[merged['_merge'] == 'left_only']
-```
-
-### Self-Join
-
-```python
-# Find pairs within same department
-employees = pd.DataFrame({
-    'emp_id': [1, 2, 3, 4],
-    'name': ['Alice', 'Bob', 'Charlie', 'Diana'],
-    'dept_id': [101, 101, 102, 101],
-})
-
-# Self-join to find pairs
-pairs = pd.merge(
-    employees,
-    employees,
-    on='dept_id',
-    suffixes=('_1', '_2')
-)
-# Remove self-pairs and duplicates
-pairs = pairs[pairs['emp_id_1'] < pairs['emp_id_2']]
-```
-
----
-
-## Best Practices Summary
-
-1. **Choose the right join type** - Default inner may drop data
-2. **Validate cardinality** - Use `validate` parameter
-3. **Use indicator** - Debug unexpected results
-4. **Handle duplicates** - Use meaningful suffixes
-5. **Pre-sort for performance** - Especially for large DataFrames
-6. **Reset index after operations** - Keep DataFrames usable
-7. **Check for NaN after join** - Understand unmatched rows
-
----
-
-## Anti-Patterns to Avoid
-
-```python
-# BAD: Merge without understanding cardinality
-result = pd.merge(df1, df2, on='key')  # May explode row count
-
-# GOOD: Validate relationship
-result = pd.merge(df1, df2, on='key', validate='one_to_one')
-
-# BAD: Repeated merges
-result = pd.merge(df1, df2, on='key')
-result = pd.merge(result, df3, on='key')
-result = pd.merge(result, df4, on='key')
-
-# GOOD: Chain or use reduce
-from functools import reduce
-dfs = [df1, df2, df3, df4]
-result = reduce(lambda left, right: pd.merge(left, right, on='key'), dfs)
-
-# BAD: Ignoring merge indicators
-result = pd.merge(df1, df2, on='key', how='outer')
-
-# GOOD: Check merge results
-result = pd.merge(df1, df2, on='key', how='outer', indicator=True)
-print(result['_merge'].value_counts())
-```
-
----
-
-## Related References
-
-- `dataframe-operations.md` - Filter before/after merge
-- `aggregation-groupby.md` - Aggregate before merging
-- `performance-optimization.md` - Optimize large merges
+# Merging and Joining
+
+---
+
+## Overview
+
+Combining DataFrames is essential for working with relational data. This reference covers merge, join, concat, and advanced combination strategies with pandas 2.0+.
+
+---
+
+## Merge (SQL-Style Joins)
+
+### Basic Merge
+
+```python
+import pandas as pd
+import numpy as np
+
+# Sample DataFrames
+employees = pd.DataFrame({
+    'emp_id': [1, 2, 3, 4, 5],
+    'name': ['Alice', 'Bob', 'Charlie', 'Diana', 'Eve'],
+    'dept_id': [101, 102, 101, 103, 102],
+})
+
+departments = pd.DataFrame({
+    'dept_id': [101, 102, 104],
+    'dept_name': ['Engineering', 'Sales', 'Marketing'],
+})
+
+# Inner join (default) - only matching rows
+result = pd.merge(employees, departments, on='dept_id')
+
+# Explicit how parameter
+result = pd.merge(employees, departments, on='dept_id', how='inner')
+```
+
+### Join Types
+
+```python
+# Inner join - only matching rows from both
+inner = pd.merge(employees, departments, on='dept_id', how='inner')
+# Result: 4 rows (emp_id 4 has dept_id 103 which doesn't exist in departments)
+
+# Left join - all rows from left, matching from right
+left = pd.merge(employees, departments, on='dept_id', how='left')
+# Result: 5 rows (Diana has NaN for dept_name)
+
+# Right join - all rows from right, matching from left
+right = pd.merge(employees, departments, on='dept_id', how='right')
+# Result: 4 rows (Marketing has no employees, but is included)
+
+# Outer join - all rows from both
+outer = pd.merge(employees, departments, on='dept_id', how='outer')
+# Result: 6 rows (includes unmatched from both sides)
+
+# Cross join - cartesian product
+cross = pd.merge(employees, departments, how='cross')
+# Result: 15 rows (5 employees x 3 departments)
+```
+
+### Merging on Different Column Names
+
+```python
+employees = pd.DataFrame({
+    'emp_id': [1, 2, 3],
+    'name': ['Alice', 'Bob', 'Charlie'],
+    'department': [101, 102, 101],
+})
+
+departments = pd.DataFrame({
+    'id': [101, 102],
+    'dept_name': ['Engineering', 'Sales'],
+})
+
+# Different column names
+result = pd.merge(
+    employees,
+    departments,
+    left_on='department',
+    right_on='id'
+)
+
+# Drop duplicate column after merge
+result = result.drop('id', axis=1)
+```
+
+### Merging on Multiple Columns
+
+```python
+sales = pd.DataFrame({
+    'region': ['East', 'East', 'West', 'West'],
+    'product': ['A', 'B', 'A', 'B'],
+    'sales': [100, 150, 120, 180],
+})
+
+targets = pd.DataFrame({
+    'region': ['East', 'East', 'West'],
+    'product': ['A', 'B', 'A'],
+    'target': [90, 140, 110],
+})
+
+# Merge on multiple columns
+result = pd.merge(sales, targets, on=['region', 'product'], how='left')
+```
+
+### Merging on Index
+
+```python
+# Set index before merge
+employees_idx = employees.set_index('emp_id')
+salaries = pd.DataFrame({
+    'emp_id': [1, 2, 3, 4],
+    'salary': [80000, 75000, 70000, 65000],
+}).set_index('emp_id')
+
+# Merge on index
+result = pd.merge(employees_idx, salaries, left_index=True, right_index=True)
+
+# Mix of column and index
+result = pd.merge(
+    employees,
+    salaries,
+    left_on='emp_id',
+    right_index=True
+)
+```
+
+---
+
+## Handling Duplicate Columns
+
+### Suffixes
+
+```python
+df1 = pd.DataFrame({
+    'id': [1, 2, 3],
+    'value': [10, 20, 30],
+    'date': ['2024-01-01', '2024-01-02', '2024-01-03'],
+})
+
+df2 = pd.DataFrame({
+    'id': [1, 2, 3],
+    'value': [100, 200, 300],
+    'date': ['2024-02-01', '2024-02-02', '2024-02-03'],
+})
+
+# Default suffixes
+result = pd.merge(df1, df2, on='id')
+# Columns: id, value_x, date_x, value_y, date_y
+
+# Custom suffixes
+result = pd.merge(df1, df2, on='id', suffixes=('_jan', '_feb'))
+# Columns: id, value_jan, date_jan, value_feb, date_feb
+```
+
+### Validate Merge Cardinality
+
+```python
+# Validate merge relationships (pandas 2.0+)
+# Raises MergeError if validation fails
+
+# One-to-one: each key appears at most once in both DataFrames
+result = pd.merge(df1, df2, on='id', validate='one_to_one')  # or '1:1'
+
+# One-to-many: keys unique in left only
+result = pd.merge(employees, salaries, on='emp_id', validate='one_to_many')  # or '1:m'
+
+# Many-to-one: keys unique in right only
+result = pd.merge(salaries, employees, on='emp_id', validate='many_to_one')  # or 'm:1'
+
+# Many-to-many: no uniqueness requirement (default)
+result = pd.merge(df1, df2, on='id', validate='many_to_many')  # or 'm:m'
+```
+
+### Indicator Column
+
+```python
+# Add indicator column showing source of each row
+result = pd.merge(
+    employees,
+    departments,
+    on='dept_id',
+    how='outer',
+    indicator=True
+)
+# _merge column values: 'left_only', 'right_only', 'both'
+
+# Custom indicator name
+result = pd.merge(
+    employees,
+    departments,
+    on='dept_id',
+    how='outer',
+    indicator='source'
+)
+
+# Filter by indicator
+left_only = result[result['_merge'] == 'left_only']
+both = result[result['_merge'] == 'both']
+```
+
+---
+
+## Join (Index-Based)
+
+### DataFrame.join()
+
+```python
+# join() is for index-based joining (simpler syntax)
+employees = pd.DataFrame({
+    'name': ['Alice', 'Bob', 'Charlie'],
+    'dept_id': [101, 102, 101],
+}, index=[1, 2, 3])
+
+salaries = pd.DataFrame({
+    'salary': [80000, 75000, 70000],
+    'bonus': [5000, 4000, 3500],
+}, index=[1, 2, 3])
+
+# Join on index
+result = employees.join(salaries)
+
+# Join types (same as merge)
+result = employees.join(salaries, how='left')
+result = employees.join(salaries, how='outer')
+```
+
+### Join on Column to Index
+
+```python
+employees = pd.DataFrame({
+    'name': ['Alice', 'Bob', 'Charlie'],
+    'dept_id': [101, 102, 101],
+})
+
+departments = pd.DataFrame({
+    'dept_name': ['Engineering', 'Sales'],
+}, index=[101, 102])
+
+# Join left column to right index
+result = employees.join(departments, on='dept_id')
+```
+
+### Join Multiple DataFrames
+
+```python
+df1 = pd.DataFrame({'a': [1, 2]}, index=['x', 'y'])
+df2 = pd.DataFrame({'b': [3, 4]}, index=['x', 'y'])
+df3 = pd.DataFrame({'c': [5, 6]}, index=['x', 'y'])
+
+# Join multiple at once
+result = df1.join([df2, df3])
+
+# With suffixes for duplicate columns
+result = df1.join([df2, df3], lsuffix='_1', rsuffix='_2')
+```
+
+---
+
+## Concat (Stacking DataFrames)
+
+### Vertical Concatenation (Row-wise)
+
+```python
+# Stack DataFrames vertically
+df1 = pd.DataFrame({
+    'name': ['Alice', 'Bob'],
+    'age': [25, 30],
+})
+
+df2 = pd.DataFrame({
+    'name': ['Charlie', 'Diana'],
+    'age': [35, 28],
+})
+
+# Basic concat (axis=0 is default)
+result = pd.concat([df1, df2])
+
+# Reset index
+result = pd.concat([df1, df2], ignore_index=True)
+
+# Keep track of source
+result = pd.concat([df1, df2], keys=['source1', 'source2'])
+# Creates MultiIndex
+```
+
+### Horizontal Concatenation (Column-wise)
+
+```python
+names = pd.DataFrame({'name': ['Alice', 'Bob', 'Charlie']})
+ages = pd.DataFrame({'age': [25, 30, 35]})
+salaries = pd.DataFrame({'salary': [50000, 60000, 70000]})
+
+# Concat columns (axis=1)
+result = pd.concat([names, ages, salaries], axis=1)
+```
+
+### Handling Mismatched Columns
+
+```python
+df1 = pd.DataFrame({
+    'name': ['Alice', 'Bob'],
+    'age': [25, 30],
+})
+
+df2 = pd.DataFrame({
+    'name': ['Charlie', 'Diana'],
+    'salary': [70000, 65000],
+})
+
+# Outer join (default) - include all columns
+result = pd.concat([df1, df2])
+# age and salary columns have NaN where not present
+
+# Inner join - only common columns
+result = pd.concat([df1, df2], join='inner')
+# Only 'name' column
+```
+
+### Concat with Verification
+
+```python
+# Verify no index overlap
+try:
+    result = pd.concat([df1, df2], verify_integrity=True)
+except ValueError as e:
+    print(f"Index overlap detected: {e}")
+
+# Alternative: use ignore_index
+result = pd.concat([df1, df2], ignore_index=True)
+```
+
+---
+
+## Combine and Update
+
+### combine_first() - Fill Gaps
+
+```python
+# Fill NaN values from another DataFrame
+df1 = pd.DataFrame({
+    'A': [1, np.nan, 3],
+    'B': [np.nan, 2, 3],
+}, index=['a', 'b', 'c'])
+
+df2 = pd.DataFrame({
+    'A': [10, 20, 30],
+    'B': [10, 20, 30],
+}, index=['a', 'b', 'c'])
+
+# Fill NaN in df1 with values from df2
+result = df1.combine_first(df2)
+# A: [1, 20, 3], B: [10, 2, 3]
+```
+
+### update() - In-Place Update
+
+```python
+df1 = pd.DataFrame({
+    'A': [1, 2, 3],
+    'B': [4, 5, 6],
+}, index=['a', 'b', 'c'])
+
+df2 = pd.DataFrame({
+    'A': [10, 20],
+    'B': [40, 50],
+}, index=['a', 'b'])
+
+# Update df1 with values from df2 (in-place)
+df1.update(df2)
+# df1 now has A: [10, 20, 3], B: [40, 50, 6]
+
+# Only update where df2 has non-NaN
+df1.update(df2, overwrite=False)  # Don't overwrite existing values
+```
+
+---
+
+## Advanced Merge Patterns
+
+### Merge with Aggregation
+
+```python
+# Merge and aggregate in one operation
+orders = pd.DataFrame({
+    'order_id': [1, 2, 3, 4],
+    'customer_id': [101, 102, 101, 103],
+    'amount': [100, 200, 150, 300],
+})
+
+customers = pd.DataFrame({
+    'customer_id': [101, 102, 103],
+    'name': ['Alice', 'Bob', 'Charlie'],
+})
+
+# Get customer summary
+customer_summary = orders.groupby('customer_id').agg(
+    total_orders=('order_id', 'count'),
+    total_amount=('amount', 'sum'),
+).reset_index()
+
+# Merge with customer info
+result = pd.merge(customers, customer_summary, on='customer_id')
+```
+
+### Merge Asof (Nearest Match)
+
+```python
+# Merge on nearest key (useful for time series)
+trades = pd.DataFrame({
+    'time': pd.to_datetime(['2024-01-01 10:00:01', '2024-01-01 10:00:03', '2024-01-01 10:00:05']),
+    'ticker': ['AAPL', 'AAPL', 'AAPL'],
+    'price': [150.0, 151.0, 150.5],
+})
+
+quotes = pd.DataFrame({
+    'time': pd.to_datetime(['2024-01-01 10:00:00', '2024-01-01 10:00:02', '2024-01-01 10:00:04']),
+    'ticker': ['AAPL', 'AAPL', 'AAPL'],
+    'bid': [149.5, 150.5, 150.0],
+    'ask': [150.5, 151.5, 151.0],
+})
+
+# Merge asof - find nearest quote for each trade
+result = pd.merge_asof(
+    trades.sort_values('time'),
+    quotes.sort_values('time'),
+    on='time',
+    by='ticker',
+    direction='backward'  # Use most recent quote
+)
+```
+
+### Conditional Merge
+
+```python
+# Merge with conditions beyond key equality
+# First merge, then filter
+
+products = pd.DataFrame({
+    'product_id': [1, 2, 3],
+    'name': ['Widget', 'Gadget', 'Gizmo'],
+    'category': ['A', 'B', 'A'],
+})
+
+discounts = pd.DataFrame({
+    'category': ['A', 'A', 'B'],
+    'min_qty': [10, 50, 20],
+    'discount': [0.05, 0.10, 0.08],
+})
+
+# Cross merge then filter
+merged = pd.merge(products, discounts, on='category')
+# Then apply quantity-based filtering as needed
+```
+
+---
+
+## Performance Considerations
+
+### Pre-sorting for Merge
+
+```python
+# Sort keys before merge for better performance
+df1 = df1.sort_values('key')
+df2 = df2.sort_values('key')
+
+# Merge sorted DataFrames
+result = pd.merge(df1, df2, on='key')
+```
+
+### Index Alignment
+
+```python
+# Using index for merge is often faster than columns
+df1 = df1.set_index('key')
+df2 = df2.set_index('key')
+
+# Join on index
+result = df1.join(df2)
+```
+
+### Memory-Efficient Merge
+
+```python
+# For large DataFrames, reduce memory before merge
+# Convert to appropriate types
+df1['key'] = df1['key'].astype('int32')  # Instead of int64
+df1['category'] = df1['category'].astype('category')
+
+# Select only needed columns
+cols_needed = ['key', 'value1', 'value2']
+result = pd.merge(df1[cols_needed], df2[cols_needed], on='key')
+```
+
+---
+
+## Common Merge Patterns
+
+### Left Join with Null Check
+
+```python
+# Find unmatched rows after left join
+result = pd.merge(employees, departments, on='dept_id', how='left')
+unmatched = result[result['dept_name'].isna()]
+```
+
+### Anti-Join (Rows Not in Other)
+
+```python
+# Find employees NOT in a specific department list
+dept_list = [101, 102]
+
+# Method 1: Using isin
+not_in_depts = employees[~employees['dept_id'].isin(dept_list)]
+
+# Method 2: Using merge with indicator
+merged = pd.merge(
+    employees,
+    pd.DataFrame({'dept_id': dept_list}),
+    on='dept_id',
+    how='left',
+    indicator=True
+)
+not_in_depts = merged[merged['_merge'] == 'left_only']
+```
+
+### Self-Join
+
+```python
+# Find pairs within same department
+employees = pd.DataFrame({
+    'emp_id': [1, 2, 3, 4],
+    'name': ['Alice', 'Bob', 'Charlie', 'Diana'],
+    'dept_id': [101, 101, 102, 101],
+})
+
+# Self-join to find pairs
+pairs = pd.merge(
+    employees,
+    employees,
+    on='dept_id',
+    suffixes=('_1', '_2')
+)
+# Remove self-pairs and duplicates
+pairs = pairs[pairs['emp_id_1'] < pairs['emp_id_2']]
+```
+
+---
+
+## Best Practices Summary
+
+1. **Choose the right join type** - Default inner may drop data
+2. **Validate cardinality** - Use `validate` parameter
+3. **Use indicator** - Debug unexpected results
+4. **Handle duplicates** - Use meaningful suffixes
+5. **Pre-sort for performance** - Especially for large DataFrames
+6. **Reset index after operations** - Keep DataFrames usable
+7. **Check for NaN after join** - Understand unmatched rows
+
+---
+
+## Anti-Patterns to Avoid
+
+```python
+# BAD: Merge without understanding cardinality
+result = pd.merge(df1, df2, on='key')  # May explode row count
+
+# GOOD: Validate relationship
+result = pd.merge(df1, df2, on='key', validate='one_to_one')
+
+# BAD: Repeated merges
+result = pd.merge(df1, df2, on='key')
+result = pd.merge(result, df3, on='key')
+result = pd.merge(result, df4, on='key')
+
+# GOOD: Chain or use reduce
+from functools import reduce
+dfs = [df1, df2, df3, df4]
+result = reduce(lambda left, right: pd.merge(left, right, on='key'), dfs)
+
+# BAD: Ignoring merge indicators
+result = pd.merge(df1, df2, on='key', how='outer')
+
+# GOOD: Check merge results
+result = pd.merge(df1, df2, on='key', how='outer', indicator=True)
+print(result['_merge'].value_counts())
+```
+
+---
+
+## Related References
+
+- `dataframe-operations.md` - Filter before/after merge
+- `aggregation-groupby.md` - Aggregate before merging
+- `performance-optimization.md` - Optimize large merges